ЛР4 - БВТ2302 - Федотов - на 3 #52

Closed
imenno_on wants to merge 1 commits from imenno_on/mtusi2026lab4-fork:solution/BVT2302-Fedotov into main
77 changed files with 2300 additions and 52 deletions
+15 -15
View File
@@ -1,20 +1,20 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Copy to .env and replace every placeholder with a local random value.
POSTGRES_USER=messenger
POSTGRES_PASSWORD=replace_with_random_hex
POSTGRES_DB=messenger
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
DB_POOL_MIN_SIZE=1
DB_POOL_MAX_SIZE=8
DB_POOL_ACQUIRE_TIMEOUT_SECONDS=3
DB_COMMAND_TIMEOUT_SECONDS=5
REDIS_TIMEOUT_SECONDS=2
S3_ACCESS_KEY_ID=replace_with_random_access_id
S3_SECRET_ACCESS_KEY=replace_with_random_secret
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_CONNECT_TIMEOUT_SECONDS=3
S3_READ_TIMEOUT_SECONDS=10
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
API_REPLICAS=2
LB_PORT=8080
+2
View File
@@ -0,0 +1,2 @@
* text=auto
*.sh text eol=lf
+17 -14
View File
@@ -1,18 +1,21 @@
## Лабораторная 4
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Федотов Михаил, БВТ2302
Целевая оценка: 3
Трек 5: не заявлен
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и перенесено: обязательная основа Lab 3 (PostgreSQL, Argon2id и Redis-сессии, LB × 2 API, идемпотентность) и Lab 4 (RustFS S3, versioned migration 0004, metadata в PostgreSQL, оригиналы в приватном bucket, authenticated upload/download, attachment_ids в сообщениях). Собственные проверки размещены в student-tests/.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Команды запуска из чистого clone: см. раздел «Запуск из чистого clone» в [REPORT.md](../REPORT.md). Кратко: скопировать .env.example в .env, заменить secrets, выполнить docker compose config и up --build, затем make check/validate/test и student tests.
Результаты проверок: make check, make validate PYTHON=.venv/bin/python, make test, student tests, S3 outage/recovery, restart, API failover и повторный запуск миграций прошли. Конкретный stdout лежит в evidence/; сводка результатов и ограничения — в REPORT.md.
Ограничения / известные ошибки: одноузловый RustFS и один Docker host; автоматическая сверка orphan S3 objects и отдельная минимальная S3 policy не реализованы, так как целевой уровень — 3. TLS и отдельный load benchmark не заявлены; стенд рассчитан на localhost и синтетические данные.
Источники и использование ИИ: исходники Lab 3 — собственная предыдущая работа; RustFS использован по официальной документации. При переносе и реализации использовался ИИ-помощник.
- [x] Контракт и публичные tests сохранены без ослабления.
- [x] Отчёт содержит инструкцию запуска именно этого решения.
- [x] Secrets и локальные volumes исключены из Git через .gitignore.
- [x] make check, make validate и make test выполнены; приложен реальный вывод.
- [x] Сценарии отказа и сохранности уровня 3 воспроизведены.
+16
View File
@@ -0,0 +1,16 @@
FROM python:3.12.8-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
AWS_EC2_METADATA_DISABLED=true
WORKDIR /app
COPY requirements-app.txt ./
RUN pip install --no-cache-dir -r requirements-app.txt
COPY app ./app
COPY migrations ./migrations
RUN useradd --create-home --uid 10001 appuser
USER appuser
EXPOSE 8080
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]
+19 -4
View File
@@ -4,10 +4,10 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-student test-idempotency test-private-bucket test-s3-failure test-attachment-failover test-persistence test-failover
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "check: offline structure; validate: full OpenAPI; test: public smoke; test-student: own Lab 4 checks"
check:
$(PYTHON) scripts/check_contract.py
@@ -18,13 +18,28 @@ validate:
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-student:
$(PYTHON) student-tests/attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency:
$(PYTHON) student-tests/idempotency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-private-bucket:
$(PYTHON) student-tests/private_bucket.py
test-s3-failure:
$(PYTHON) student-tests/s3_failure.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachment-failover:
$(PYTHON) student-tests/attachment_failover.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to scripts/restart.sh"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to scripts/stop-one.sh"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+77 -19
View File
@@ -1,37 +1,95 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Федотов Михаил, БВТ2302
- Целевая оценка: 3
- Трек 5: не заявлен
- Commit/PR: ещё не создано по просьбе автора; текущая ветка `solution/BVT2302-Fedotov`
- Использованные источники и помощь: исходный код и миграции собственной Lab 3; [официальная документация RustFS](https://docs.rustfs.com/en/installation/container/docker) и [закреплённый образ 1.0.4-glibc](https://github.com/rustfs/rustfs/pkgs/container/rustfs); при разработке использовался ИИ-помощник. Публичные README, контракты и тесты Lab 4 не копировались поверх новой версии и не ослаблялись.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Engine с Compose V2 и Python 3.9+ для публичных проверок. Приложение собирается в Docker на Python 3.12.8. Подготовьте локальные значения secrets; файл `.env` игнорируется Git:
```sh
cp .env.example .env
sed -i "s/replace_with_random_hex/$(openssl rand -hex 24)/" .env
sed -i "s/replace_with_random_access_id/$(openssl rand -hex 12)/" .env
sed -i "s/replace_with_random_secret/$(openssl rand -hex 32)/" .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
until curl -fsS http://localhost:8080/health/ready; do sleep 2; done
```
Проверки:
```sh
make check
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
make test BASE_URL=http://localhost:8080
make test-student BASE_URL=http://localhost:8080
make test-idempotency BASE_URL=http://localhost:8080
make test-private-bucket
make test-s3-failure
make test-attachment-failover BASE_URL=http://localhost:8080
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080
docker compose --env-file .env up -d --scale api=2
```
Обычная остановка без удаления данных: `docker compose --env-file .env down`. Намеренное удаление учебной БД, сессий и S3-объектов: `docker compose --env-file .env down -v`; выполнять только при осознанном сбросе стенда.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Клиент → Nginx LB → две API-реплики → PostgreSQL, Redis и приватный RustFS S3. Наружу опубликован только LB на localhost:8080; PostgreSQL, Redis, API и S3 доступны только во внутренней Compose-сети. Консоль RustFS выключена и порт S3 на хост не опубликован.
Из Lab 3 перенесены FastAPI-маршруты и схема Basic → Argon2id/Bearer-сессия, PostgreSQL/Redis-хранилище, миграции 0001–0003, cursor pagination, идемпотентность сообщений, Nginx и hooks отказа/перезапуска. Публичные `tests/`, `contracts/`, README и шаблоны преподавателя не заменялись кодом из предыдущих лабораторных.
В Lab 4 добавлена миграция 0004:
- `attachments` хранит ID чата/владельца, имя для отображения, MIME, размер, SHA-256, статус и серверный object key;
- `message_attachments` связывает сообщение и вложение, а составные внешние ключи запрещают ссылку на файл другого чата;
- пустой текст сообщения разрешён только если в запросе есть хотя бы одно вложение; проверка выполняется API.
Оригинальные байты находятся только в S3. API ограниченно читает тело до 10 MiB, распознаёт PNG/JPEG через Pillow, сопоставляет MIME с реальным форматом и проверяет пиксели до декодирования. S3 объект записывается до метаданных; если запись метаданных в PostgreSQL не проходит, API пытается удалить только что записанный объект. HTTP 201 возвращается после записи объекта и метаданных. Object key генерируется как UUID и не зависит от filename.
Bucket создаётся при старте API, отдельная политика публичного доступа не назначается; S3 endpoint не опубликован наружу. Участник чата получает metadata и оригинал через API, посторонний получает 404. Все сервисы состояния используют именованные volumes: `postgres_data`, `redis_data`, `s3_data`.
Образы закреплены: Python 3.12.8-slim, PostgreSQL 16.4-alpine, Redis 7.4.2-alpine, Nginx 1.27.4-alpine и RustFS 1.0.4-glibc по digest. Версии Python-зависимостей приложения закреплены в `requirements-app.txt`.
| Роль | Service | Данные/готовность | Порт и credentials |
| --- | --- | --- | --- |
| API | `api`, 2 реплики | readiness проверяет PostgreSQL, Redis и S3; локального состояния нет | доступ только из Compose-сети |
| Метаданные | `postgres`, 16.4 | именованный volume; `pg_isready` | внутренний 5432; пароль только в локальном `.env` |
| Сессии | `redis`, 7.4.2 | AOF и `appendfsync always`, именованный volume | внутренний 6379 |
| Бинарные объекты | `s3`, RustFS 1.0.4 | приватный bucket, именованный volume; API проверяет `head_bucket` | внутренний 9000; console выключена |
| Балансировщик | `lb`, Nginx 1.27.4 | healthcheck через API readiness | localhost:8080 |
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Критерий | Проверка | Результат финального прогона |
| --- | --- | --- |
| Структура и сохранность контракта | `make check`, `git diff --check` | PASS; `evidence/make-check.txt`, `evidence/diff-check.txt` |
| OpenAPI 3.1 и JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS; `evidence/make-validate.txt` |
| Синтаксис приложения, тестов и hooks | `compileall`, YAML parse, `sh -n`, Compose config | PASS; отдельные stdout-файлы в `evidence/` |
| Публичный API и вложения | `make test`, `make test-student` | PASS; `evidence/make-test.txt`, `evidence/test-student.txt` |
| Идемпотентность и доступ к bucket | `make test-idempotency`, `make test-private-bucket` | PASS; 16 конкурентных запросов, приватный bucket отвечает 403 на анонимный листинг |
| Отказ S3 и чтение между репликами | `make test-s3-failure`, `make test-attachment-failover` | PASS; API вернул 503 при отказе S3, метаданные сохранились, файл читается после восстановления и через вторую API-реплику |
| Перезапуск и API failover | `make test-persistence`, `make test-failover` | PASS; подтверждённые данные и действующая сессия сохранены |
| Повторный запуск миграций | два запуска `migrate` и запрос `schema_migrations` | PASS; в таблице ровно версии 1, 2, 3, 4 |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Первый запуск нового S3-теста оборвался по 5-секундному таймауту тестового HTTP-клиента до завершения повторных попыток S3 SDK. Причина и исправление тестового таймаута записаны в `evidence/test-s3-failure-analysis.txt`; следующий прогон прошёл полностью. Ошибки приложения этим первым запуском не маскируются.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
`scripts/restart.sh` пересоздаёт контейнеры и сеть Compose, не удаляя volumes. Финальный `make test-persistence` подтвердил чтение существующих чатов/сообщений, действующей сессии и оригинала вложения после перезапуска стека.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
`scripts/stop-one.sh` находит API контейнер по `X-Instance-Id` и останавливает только его. Межрепличный тест загрузил вложение, остановил обслужившую запрос API-реплику и прочитал metadata и оригинальные байты через оставшуюся реплику. Нагрузочный failover подтвердил 2 реплики до отказа, 140 успешных записей, 0 неуспешных попыток и восстановление за 0.556 секунды; подтверждённая запись и действующая сессия сохранились. Финальный статус контейнеров записан в `evidence/compose-final.txt`.
## Ограничения и следующие шаги
Тест отказа S3 остановил только service `s3`: readiness и чтение оригинала вернули ограниченный HTTP 503, PostgreSQL metadata оставались доступны, после перезапуска S3 исходные байты читались без изменений. Проверка неавторизованного доступа к bucket получила HTTP 403.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Аппаратный профиль WSL на момент проверки: 32 CPU threads и 15 GiB памяти; Docker Engine 29.8.0. Для каждой API-реплики установлены пределы 1 CPU и 512 MiB; для остальных контейнеров явные resource limits не заданы. Отдельный throughput/load benchmark не запускался и не заявляется. Все фактические stdout-проверки и профиль сохранены в `evidence/`.
## Ограничения
Цель этой работы — уровень 3. Один узел RustFS и один Docker host сами по себе не обеспечивают отказоустойчивость к потере диска/хоста. API использует ключ доступа RustFS из локального `.env`; отдельный пользователь S3 с минимальными правами и механизм сверки orphan объектов относятся к дополнительным практикам уровня 4 и не заявляются. Если компенсирующее удаление после ошибки PostgreSQL также не удастся, может остаться orphan-объект; автоматическая сверка не реализована. TLS отсутствует, стенд предназначен для localhost и синтетических данных.
View File
View File
+42
View File
@@ -0,0 +1,42 @@
from fastapi import Request
from app.core.errors import ApiError
from app.repositories.memory import SessionRow, UserRow
import uuid as _uuid
BEARER_CHALLENGE = {'WWW-Authenticate': 'Bearer realm="messenger"'}
def storage_of(request: Request):
return request.app.state.storage
async def current_session(request: Request) -> SessionRow:
header = request.headers.get('authorization', '')
token = header[7:].strip() if header[:7].lower() == 'bearer ' else ''
if not token:
raise ApiError(401, 'Bearer session required', headers=BEARER_CHALLENGE)
row = await storage_of(request).session(token)
if row is None:
raise ApiError(401, 'Session is missing, expired or revoked', headers=BEARER_CHALLENGE)
return row
async def current_user(request: Request) -> UserRow:
session = await current_session(request)
user = await storage_of(request).user(session.user_id)
if user is None:
raise ApiError(401, 'Session user no longer exists', headers=BEARER_CHALLENGE)
return user
async def require_chat_member(request: Request, chat_id: str, user: UserRow):
try:
_uuid.UUID(chat_id)
except ValueError:
raise ApiError(400, 'chat_id must be a UUID')
row = await storage_of(request).chat(chat_id)
if row is None or user.id not in row.member_ids:
raise ApiError(404, 'Chat is not available for this user')
return row
View File
+155
View File
@@ -0,0 +1,155 @@
import hashlib
import io
import uuid
import warnings
from fastapi import APIRouter, Query, Request
from fastapi.responses import Response
from PIL import Image, UnidentifiedImageError
from app.api.deps import current_user, require_chat_member
from app.core.config import settings
from app.core.errors import ApiError
router = APIRouter()
MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
def _validate_filename(filename: str) -> None:
if not filename or len(filename) > 255 or not filename.strip():
raise ApiError(400, 'Invalid filename')
if any(ch in filename for ch in ('/', chr(92))):
raise ApiError(400, 'Invalid filename')
if any(ord(ch) == 0 or __import__('unicodedata').category(ch).startswith('C') for ch in filename):
raise ApiError(400, 'Invalid filename')
async def _read_bounded_body(request: Request) -> bytes:
length = request.headers.get('content-length')
if length:
try:
size = int(length)
except ValueError:
raise ApiError(400, 'Invalid Content-Length') from None
if size < 0:
raise ApiError(400, 'Invalid Content-Length')
if size > settings.max_upload_bytes:
raise ApiError(413, 'Image exceeds the maximum size')
chunks = []
total = 0
async for chunk in request.stream():
total += len(chunk)
if total > settings.max_upload_bytes:
raise ApiError(413, 'Image exceeds the maximum size')
chunks.append(chunk)
if total == 0:
raise ApiError(415, 'Empty or invalid image')
return b''.join(chunks)
def _validate_image(data: bytes, declared_type: str) -> None:
if declared_type not in {'image/png', 'image/jpeg'}:
raise ApiError(415, 'Only PNG and JPEG images are accepted')
try:
with warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
with Image.open(io.BytesIO(data)) as image:
actual_type = MIME_BY_FORMAT.get(image.format)
width, height = image.size
if actual_type is None or actual_type != declared_type:
raise ApiError(415, 'Image content does not match Content-Type')
if width < 1 or height < 1:
raise ApiError(415, 'Invalid image dimensions')
if width * height > settings.max_image_pixels:
raise ApiError(413, 'Image exceeds the maximum pixel count')
image.verify()
except ApiError:
raise
except (UnidentifiedImageError, Image.DecompressionBombError,
Image.DecompressionBombWarning, OSError, SyntaxError, ValueError):
raise ApiError(415, 'Image content is invalid or unsupported') from None
def attachment_out(row) -> dict:
return {
'id': row.id,
'chat_id': row.chat_id,
'owner_id': row.owner_id,
'filename': row.filename,
'content_type': row.content_type,
'size_bytes': row.size_bytes,
'sha256': row.sha256,
'status': 'ready',
'variants': [],
'error_code': None,
'created_at': row.created_at,
'updated_at': row.updated_at,
}
@router.post('/api/v1/chats/{chat_id}/attachments', status_code=201)
async def upload_attachment(
chat_id: str,
request: Request,
filename: str = Query(min_length=1, max_length=255),
):
user = await current_user(request)
chat = await require_chat_member(request, chat_id, user)
_validate_filename(filename)
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
if content_type not in {'image/png', 'image/jpeg'}:
raise ApiError(415, 'Only PNG and JPEG images are accepted')
data = await _read_bounded_body(request)
_validate_image(data, content_type)
attachment_id = str(uuid.uuid4())
object_key = 'original/' + uuid.uuid4().hex
digest = hashlib.sha256(data).hexdigest()
object_store = request.app.state.object_store
await object_store.put(object_key, data, content_type)
try:
row = await request.app.state.storage.create_attachment(
attachment_id=attachment_id,
chat_id=chat.id,
owner_id=user.id,
object_key=object_key,
filename=filename,
content_type=content_type,
size_bytes=len(data),
sha256=digest,
)
except Exception:
# Если запись метаданных не зафиксирована, пытаемся убрать только что загруженный объект.
try:
await object_store.delete(object_key)
except ApiError:
pass
raise
return attachment_out(row)
@router.get('/api/v1/attachments/{attachment_id}')
async def get_attachment(attachment_id: str, request: Request):
user = await current_user(request)
row = await request.app.state.storage.attachment_for_member(attachment_id, user.id)
if row is None:
raise ApiError(404, 'Attachment is not available for this user')
return attachment_out(row)
@router.get('/api/v1/attachments/{attachment_id}/content')
async def download_attachment(
attachment_id: str,
request: Request,
variant: str = Query(default='original', pattern=r'^original$'),
):
user = await current_user(request)
row = await request.app.state.storage.attachment_for_member(attachment_id, user.id)
if row is None:
raise ApiError(404, 'Attachment is not available for this user')
data = await request.app.state.object_store.get(row.object_key)
return Response(
content=data,
media_type=row.content_type,
headers={'Cache-Control': 'private, no-store'},
)
+31
View File
@@ -0,0 +1,31 @@
from fastapi import APIRouter, Request
from fastapi.responses import Response
from app.api.deps import current_session
from app.api.serialize import user_out
from app.core.config import settings
from app.core.errors import ApiError
from app.core.security import parse_basic, verify_password
from app.repositories.memory import iso_z
router = APIRouter()
BASIC_CHALLENGE = {'WWW-Authenticate': 'Basic realm="messenger", charset="UTF-8"'}
@router.post('/api/v1/auth/sessions', status_code=201)
async def login(request: Request):
creds = parse_basic(request.headers.get('authorization', ''))
storage = request.app.state.storage
row = await storage.user_by_username(creds[0]) if creds else None
if creds is None or row is None or not verify_password(row.password_hash, creds[1]):
raise ApiError(401, 'Invalid credentials', headers=BASIC_CHALLENGE)
session = await storage.create_session(row.id, settings.session_ttl_seconds)
return {'token': session.token, 'token_type': 'Bearer',
'expires_at': iso_z(session.expires_at), 'user': user_out(row)}
@router.delete('/api/v1/auth/sessions/current', status_code=204)
async def logout(request: Request):
session = await current_session(request)
await request.app.state.storage.revoke(session.token)
return Response(status_code=204)
+33
View File
@@ -0,0 +1,33 @@
from fastapi import APIRouter, Query, Request
from app.api.deps import current_user, require_chat_member
from app.api.serialize import chat_out
from app.core.cursors import decode_cursor, paginate
from app.schemas import CreateChat
router = APIRouter()
@router.post('/api/v1/chats', status_code=201)
async def create_chat(body: CreateChat, request: Request):
user = await current_user(request)
member_ids = [str(mid) for mid in body.member_ids]
row = await request.app.state.storage.create_chat(body.title, member_ids, user.id)
return chat_out(row)
@router.get('/api/v1/chats')
async def list_chats(request: Request, limit: int = Query(50, ge=1, le=100),
cursor: str | None = Query(None, min_length=1)):
user = await current_user(request)
resource = 'chats:' + user.id
key = decode_cursor(cursor, resource) if cursor else None
rows = await request.app.state.storage.chats_of(user.id)
page, next_cursor = paginate(rows, key, limit, resource)
return {'items': [chat_out(r) for r in page], 'next_cursor': next_cursor}
@router.get('/api/v1/chats/{chat_id}')
async def get_chat(chat_id: str, request: Request):
user = await current_user(request)
return chat_out(await require_chat_member(request, chat_id, user))
+17
View File
@@ -0,0 +1,17 @@
from fastapi import APIRouter, Request
from app.core.errors import ApiError
router = APIRouter()
@router.get('/health/live')
async def live():
return {'status': 'ok'}
@router.get('/health/ready')
async def ready(request: Request):
if not await request.app.state.storage.ready():
raise ApiError(503, 'Storage is unavailable')
return {'status': 'ok'}
+11
View File
@@ -0,0 +1,11 @@
from fastapi import APIRouter, Request
from app.api.deps import current_user
from app.api.serialize import user_out
router = APIRouter()
@router.get('/api/v1/users/me')
async def me(request: Request):
return user_out(await current_user(request))
+52
View File
@@ -0,0 +1,52 @@
import hashlib
import json
from fastapi import APIRouter, Header, Query, Request
from app.api.deps import current_user, require_chat_member
from app.api.serialize import message_out
from app.core.cursors import decode_cursor, encode_cursor
from app.core.errors import ApiError
from app.schemas import CreateMessage
router = APIRouter()
@router.post('/api/v1/chats/{chat_id}/messages', status_code=201)
async def create_message(
body: CreateMessage,
chat_id: str,
request: Request,
idempotency_key: str | None = Header(
default=None, alias='Idempotency-Key', min_length=1, max_length=128,
pattern=r'^[A-Za-z0-9._:-]{1,128}$',
),
):
if idempotency_key is None:
raise ApiError(400, 'Idempotency-Key is required')
if not body.text.strip() and not body.attachment_ids:
raise ApiError(400, 'Message must contain text or at least one attachment')
user = await current_user(request)
chat = await require_chat_member(request, chat_id, user)
normalized_body = json.dumps(
body.model_dump(mode='json'), ensure_ascii=False, sort_keys=True, separators=(',', ':')
).encode('utf-8')
body_hash = hashlib.sha256(normalized_body).digest()
row = await request.app.state.storage.add_idempotent_message(
chat.id, user.id, body.text, [str(value) for value in body.attachment_ids],
idempotency_key, body_hash,
)
return message_out(row)
@router.get('/api/v1/chats/{chat_id}/messages')
async def list_messages(request: Request, chat_id: str,
limit: int = Query(50, ge=1, le=100),
cursor: str | None = Query(None, min_length=1)):
user = await current_user(request)
chat = await require_chat_member(request, chat_id, user)
resource = 'messages:' + chat.id
key = decode_cursor(cursor, resource) if cursor else None
rows, has_more = await request.app.state.storage.messages_page(chat.id, key, limit)
next_cursor = encode_cursor(resource, rows[-1].created_at, rows[-1].id) if has_more else None
return {'items': [message_out(row) for row in rows], 'next_cursor': next_cursor}
+14
View File
@@ -0,0 +1,14 @@
from fastapi import APIRouter, Request
from app.api.serialize import user_out
from app.core.security import hash_password
from app.schemas import CreateUser
router = APIRouter()
@router.post('/api/v1/users', status_code=201)
async def register(body: CreateUser, request: Request):
row = await request.app.state.storage.create_user(
body.username, body.display_name, hash_password(body.password))
return user_out(row)
+17
View File
@@ -0,0 +1,17 @@
from app.repositories.memory import UserRow
def user_out(row: UserRow) -> dict:
return {'id': row.id, 'username': row.username,
'display_name': row.display_name, 'created_at': row.created_at}
def chat_out(row) -> dict:
return {'id': row.id, 'title': row.title,
'member_ids': list(row.member_ids), 'created_at': row.created_at}
def message_out(row) -> dict:
return {'id': row.id, 'chat_id': row.chat_id, 'sender_id': row.sender_id,
'text': row.text, 'created_at': row.created_at,
'attachment_ids': list(row.attachment_ids)}
View File
+55
View File
@@ -0,0 +1,55 @@
from pathlib import Path
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file='.env', extra='ignore')
app_host: str = '0.0.0.0'
app_port: int = 8080
session_ttl_seconds: int = 3600
database_url: str = ''
database_url_file: str | None = None
redis_url: str = 'redis://localhost:6379/0'
redis_url_file: str | None = None
db_pool_min_size: int = 1
db_pool_max_size: int = 10
db_pool_acquire_timeout_seconds: float = 3.0
db_command_timeout_seconds: float = 5.0
redis_timeout_seconds: float = 2.0
s3_endpoint_url: str = 'http://localhost:9000'
s3_region: str = 'us-east-1'
s3_bucket: str = 'messenger-attachments'
s3_access_key_id: str = ''
s3_access_key_id_file: str | None = None
s3_secret_access_key: str = ''
s3_secret_access_key_file: str | None = None
s3_connect_timeout_seconds: float = 3.0
s3_read_timeout_seconds: float = 10.0
max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760)
max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000)
def resolved_database_url(self) -> str:
if self.database_url_file:
return Path(self.database_url_file).read_text(encoding='utf-8').strip()
return self.database_url
def resolved_redis_url(self) -> str:
if self.redis_url_file:
return Path(self.redis_url_file).read_text(encoding='utf-8').strip()
return self.redis_url
def resolved_s3_access_key_id(self) -> str:
if self.s3_access_key_id_file:
return Path(self.s3_access_key_id_file).read_text(encoding='utf-8').strip()
return self.s3_access_key_id
def resolved_s3_secret_access_key(self) -> str:
if self.s3_secret_access_key_file:
return Path(self.s3_secret_access_key_file).read_text(encoding='utf-8').strip()
return self.s3_secret_access_key
settings = Settings()
+37
View File
@@ -0,0 +1,37 @@
import base64
import binascii
import json
import uuid
from datetime import datetime
from app.core.errors import ApiError
from app.repositories.memory import order_key
def encode_cursor(resource: str, created_at: str, row_id: str) -> str:
payload = json.dumps({'r': resource, 't': created_at, 'i': row_id}, separators=(',', ':'))
return base64.urlsafe_b64encode(payload.encode()).decode()
def decode_cursor(value: str, resource: str) -> tuple[datetime, int]:
"""Курсор от другого ресурса или битый -> 400 invalid_request."""
try:
payload = json.loads(base64.urlsafe_b64decode(value.encode()))
if payload['r'] != resource:
raise ValueError('cursor belongs to another resource')
moment = datetime.fromisoformat(payload['t'].replace('Z', '+00:00'))
return moment, uuid.UUID(payload['i']).int
except (ValueError, KeyError, TypeError, binascii.Error):
raise ApiError(400, 'Invalid cursor')
def paginate(rows: list, cursor_key, limit: int, resource: str):
"""Keyset: берём строки строго после (created_at, id) курсора, без offset-скана."""
if cursor_key is not None:
rows = [r for r in rows if order_key(r.created_dt, r.id) > cursor_key]
page = rows[:limit]
next_cursor = None
if len(rows) > limit: # следующая запись есть на момент чтения
last = page[-1]
next_cursor = encode_cursor(resource, last.created_at, last.id)
return page, next_cursor
+72
View File
@@ -0,0 +1,72 @@
import os
import socket
import uuid
import asyncpg
from redis.exceptions import RedisError
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
from starlette.middleware.base import BaseHTTPMiddleware
CODES = {400: 'invalid_request', 401: 'unauthorized', 404: 'not_found', 409: 'conflict',
413: 'payload_too_large', 415: 'unsupported_media_type', 429: 'rate_limited',
503: 'unavailable', 500: 'internal_error'}
class ApiError(Exception):
"""Бизнес-ошибка: HTTP-статус + код контракта + безопасное сообщение."""
def __init__(self, status_code: int, message: str, code: str | None = None,
headers: dict | None = None):
self.status_code = status_code
self.code = code or CODES.get(status_code, 'internal_error')
self.message = message
self.headers = headers or {}
class RequestIdMiddleware(BaseHTTPMiddleware):
"""Каждый ответ, включая 204 и ошибки, несёт непустой X-Request-Id."""
async def dispatch(self, request: Request, call_next):
request.state.request_id = str(uuid.uuid4())
response = await call_next(request)
response.headers['X-Request-Id'] = request.state.request_id
response.headers['X-Instance-Id'] = os.environ.get('INSTANCE_ID') or socket.gethostname()
return response
def _body(request: Request, status_code: int, code: str, message: str,
headers: dict | None = None) -> JSONResponse:
rid = getattr(request.state, 'request_id', str(uuid.uuid4()))
return JSONResponse(status_code=status_code, headers=headers,
content={'error': {'code': code, 'message': message, 'request_id': rid}})
def register_error_handlers(app: FastAPI) -> None:
@app.exception_handler(ApiError)
async def on_api_error(request: Request, exc: ApiError):
return _body(request, exc.status_code, exc.code, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def on_validation(request: Request, exc: RequestValidationError):
return _body(request, 400, 'invalid_request', 'Request validation failed')
@app.exception_handler(StarletteHTTPException)
async def on_http(request: Request, exc: StarletteHTTPException):
code = CODES.get(exc.status_code, 'invalid_request')
return _body(request, exc.status_code, code, 'Route or method not found')
async def on_dependency_failure(request: Request, exc: Exception):
return _body(request, 503, 'unavailable', 'A required service is unavailable')
app.add_exception_handler(asyncpg.PostgresError, on_dependency_failure)
app.add_exception_handler(RedisError, on_dependency_failure)
app.add_exception_handler(OSError, on_dependency_failure)
app.add_exception_handler(TimeoutError, on_dependency_failure)
@app.exception_handler(Exception)
async def on_unexpected(request: Request, exc: Exception):
return _body(request, 500, 'internal_error', 'Internal error')
+39
View File
@@ -0,0 +1,39 @@
import base64
import binascii
import secrets
from argon2 import PasswordHasher
from argon2.exceptions import Argon2Error
_ph = PasswordHasher() # по умолчанию Argon2id
def hash_password(password: str) -> str:
return _ph.hash(password)
def verify_password(digest: str, password: str) -> bool:
try:
return _ph.verify(digest, password)
except (Argon2Error, ValueError):
return False
def new_token() -> str:
return secrets.token_hex(32) # 32 байта = 256 бит энтропии
def parse_basic(header: str) -> tuple[str, str] | None:
"""Преобразовать 'Basic base64(username:password)' в пару (username, password) или вернуть None."""
parts = header.split(' ', 1)
if len(parts) != 2 or parts[0].lower() != 'basic':
return None
try:
raw = base64.b64decode(parts[1].strip(), validate=True)
text = raw.decode('utf-8')
except (binascii.Error, ValueError, UnicodeDecodeError):
return None
if ':' not in text:
return None
username, password = text.split(':', 1) # двоеточие в пароле допустимо
return username, password
+1
View File
@@ -0,0 +1 @@
"""Настройка базы данных и миграции схемы."""
+56
View File
@@ -0,0 +1,56 @@
from pathlib import Path
import asyncpg
MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / 'migrations'
async def apply_migrations(pool: asyncpg.Pool) -> None:
"""Применить каждую нумерованную SQL-миграцию один раз в транзакции под advisory lock."""
async with pool.acquire() as connection:
await connection.execute('SELECT pg_advisory_lock($1)', 762019002)
try:
await connection.execute(
'CREATE TABLE IF NOT EXISTS schema_migrations ('
' version integer PRIMARY KEY,'
' applied_at timestamptz NOT NULL DEFAULT now())'
)
applied = {
row['version']
for row in await connection.fetch('SELECT version FROM schema_migrations')
}
for path in sorted(MIGRATIONS_DIR.glob('[0-9]*_*.sql')):
version = int(path.name.split('_', 1)[0])
if version in applied:
continue
async with connection.transaction():
await connection.execute(path.read_text(encoding='utf-8'))
await connection.execute(
'INSERT INTO schema_migrations (version) VALUES ($1)', version
)
finally:
await connection.execute('SELECT pg_advisory_unlock($1)', 762019002)
async def _main() -> None:
import asyncio
from app.core.config import settings
dsn = settings.resolved_database_url()
if not dsn:
raise RuntimeError('Set DATABASE_URL or DATABASE_URL_FILE before running migrations')
pool = await asyncpg.create_pool(
dsn,
min_size=settings.db_pool_min_size,
max_size=settings.db_pool_max_size,
timeout=settings.db_pool_acquire_timeout_seconds,
command_timeout=settings.db_command_timeout_seconds,
)
try:
await apply_migrations(pool)
finally:
await pool.close()
if __name__ == '__main__':
import asyncio
asyncio.run(_main())
+57
View File
@@ -0,0 +1,57 @@
from contextlib import asynccontextmanager
import asyncpg
from fastapi import FastAPI
from redis.asyncio import Redis
from app.api.routes import attachments, auth, health, me, users, chats, messages
from app.core.config import settings
from app.core.errors import RequestIdMiddleware, register_error_handlers
from app.repositories.object_store import S3ObjectStore
from app.repositories.postgres import PgStorage
from app.repositories.redis_sessions import RedisSessionStore
def create_app() -> FastAPI:
@asynccontextmanager
async def lifespan(app: FastAPI):
dsn = settings.resolved_database_url()
if not dsn:
raise RuntimeError('Set DATABASE_URL or DATABASE_URL_FILE before starting the API')
pool = await asyncpg.create_pool(
dsn,
min_size=settings.db_pool_min_size,
max_size=settings.db_pool_max_size,
timeout=settings.db_pool_acquire_timeout_seconds,
command_timeout=settings.db_command_timeout_seconds,
)
redis_client = Redis.from_url(
settings.resolved_redis_url(),
decode_responses=True,
socket_connect_timeout=settings.redis_timeout_seconds,
socket_timeout=settings.redis_timeout_seconds,
)
object_store = S3ObjectStore()
try:
await object_store.ensure_bucket()
app.state.object_store = object_store
app.state.storage = PgStorage(pool, RedisSessionStore(redis_client), object_store)
yield
finally:
await redis_client.aclose()
await pool.close()
app = FastAPI(title='MTUSI Messenger lab4', docs_url=None, redoc_url=None, lifespan=lifespan)
app.add_middleware(RequestIdMiddleware)
register_error_handlers(app)
app.include_router(health.router)
app.include_router(users.router)
app.include_router(me.router)
app.include_router(auth.router)
app.include_router(chats.router)
app.include_router(messages.router)
app.include_router(attachments.router)
return app
app = create_app()
View File
+148
View File
@@ -0,0 +1,148 @@
import threading
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from app.core.errors import ApiError
def now_utc() -> datetime:
return datetime.now(timezone.utc)
def iso_z(moment: datetime) -> str:
"""RFC 3339 UTC с Z и микросекундами: порядок (created_at, id) не теряет точность."""
return moment.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
def order_key(created_dt: datetime, row_id: str):
"""Контракт: порядок по (created_at, id), id сравнивается как значение UUID."""
return (created_dt, uuid.UUID(row_id).int)
@dataclass
class UserRow:
id: str
username: str
display_name: str
password_hash: str
created_at: str
@dataclass
class ChatRow:
id: str
title: str
member_ids: list
created_dt: datetime
created_at: str
@dataclass
class MessageRow:
id: str
chat_id: str
sender_id: str
text: str
created_dt: datetime
created_at: str
attachment_ids: list[str] = field(default_factory=list)
@dataclass
class AttachmentRow:
id: str
chat_id: str
owner_id: str
object_key: str
filename: str
content_type: str
size_bytes: int
sha256: str
status: str
created_at: str
updated_at: str
@dataclass
class SessionRow:
token: str
user_id: str
expires_at: datetime
class MemoryStorage:
"""Хранилище в памяти из скелета lab1. В lab2 его заменяет PgStorage."""
def __init__(self):
self._lock = threading.RLock()
self._users: dict[str, UserRow] = {}
self._by_username: dict[str, str] = {}
self._chats: dict[str, ChatRow] = {}
self._messages: dict[str, list[MessageRow]] = {}
self._sessions: dict[str, SessionRow] = {}
def ready(self) -> bool:
return True
def create_user(self, username: str, display_name: str, password_hash: str) -> UserRow:
with self._lock:
if username in self._by_username:
raise ApiError(409, 'Username is already taken')
row = UserRow(str(uuid.uuid4()), username, display_name, password_hash,
iso_z(now_utc()))
self._users[row.id] = row
self._by_username[username] = row.id
return row
def user_by_username(self, username: str) -> UserRow | None:
with self._lock:
uid = self._by_username.get(username)
return self._users.get(uid) if uid else None
def user(self, user_id: str) -> UserRow | None:
with self._lock:
return self._users.get(user_id)
def create_session(self, user_id: str, ttl_seconds: int) -> SessionRow:
from app.core.security import new_token
with self._lock:
row = SessionRow(new_token(), user_id, now_utc() + timedelta(seconds=ttl_seconds))
self._sessions[row.token] = row
return row
def session(self, token: str) -> SessionRow | None:
with self._lock:
row = self._sessions.get(token)
if row is None:
return None
if now_utc() >= row.expires_at:
del self._sessions[token]
return None
return row
def revoke(self, token: str) -> bool:
with self._lock:
return self._sessions.pop(token, None) is not None
def create_chat(self, title: str, member_ids: list, caller_id: str) -> ChatRow:
with self._lock:
if caller_id not in member_ids:
raise ApiError(400, 'Creator must be included in member_ids')
for mid in member_ids:
if mid not in self._users:
raise ApiError(400, 'Unknown member id')
moment = now_utc()
row = ChatRow(str(uuid.uuid4()), title, list(member_ids), moment, iso_z(moment))
self._chats[row.id] = row
self._messages[row.id] = []
return row
def chat(self, chat_id: str) -> ChatRow | None:
with self._lock:
return self._chats.get(chat_id)
def chats_of(self, user_id: str) -> list[ChatRow]:
with self._lock:
rows = [c for c in self._chats.values() if user_id in c.member_ids]
return sorted(rows, key=lambda c: order_key(c.created_dt, c.id))
+109
View File
@@ -0,0 +1,109 @@
import asyncio
import time
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from app.core.config import settings
from app.core.errors import ApiError
class S3ObjectStore:
"""Хранит байты в приватном S3 bucket, не в процессе API."""
def __init__(self):
self.bucket = settings.s3_bucket
self.client = boto3.client(
"s3",
endpoint_url=settings.s3_endpoint_url,
region_name=settings.s3_region,
aws_access_key_id=settings.resolved_s3_access_key_id(),
aws_secret_access_key=settings.resolved_s3_secret_access_key(),
config=Config(
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_read_timeout_seconds,
max_pool_connections=16,
retries={"max_attempts": 2, "mode": "standard"},
s3={"addressing_style": "path"},
),
)
@staticmethod
def _unavailable():
return ApiError(503, "Object storage is unavailable")
def _ensure_bucket(self):
for attempt in range(30):
try:
self.client.head_bucket(Bucket=self.bucket)
return
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code not in {"404", "NoSuchBucket", "NotFound"}:
if code in {"403", "AccessDenied"} or attempt == 29:
raise self._unavailable() from None
else:
try:
self.client.create_bucket(Bucket=self.bucket)
except ClientError as create_exc:
create_code = str(create_exc.response.get("Error", {}).get("Code", ""))
if create_code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
if attempt == 29:
raise self._unavailable() from None
else:
return
except (BotoCoreError, OSError, TimeoutError):
if attempt == 29:
raise self._unavailable() from None
time.sleep(0.5)
raise self._unavailable()
async def ensure_bucket(self):
await asyncio.to_thread(self._ensure_bucket)
def _head_bucket(self):
self.client.head_bucket(Bucket=self.bucket)
async def ready(self) -> bool:
try:
await asyncio.to_thread(self._head_bucket)
return True
except (ClientError, BotoCoreError, OSError, TimeoutError):
return False
def _put(self, key: str, data: bytes, content_type: str):
self.client.put_object(Bucket=self.bucket, Key=key, Body=data, ContentType=content_type)
async def put(self, key: str, data: bytes, content_type: str):
try:
await asyncio.to_thread(self._put, key, data, content_type)
except (ClientError, BotoCoreError, OSError, TimeoutError):
raise self._unavailable() from None
def _get(self, key: str) -> bytes:
result = self.client.get_object(Bucket=self.bucket, Key=key)
try:
return result["Body"].read(settings.max_upload_bytes + 1)
finally:
result["Body"].close()
async def get(self, key: str) -> bytes:
try:
data = await asyncio.to_thread(self._get, key)
if len(data) > settings.max_upload_bytes:
raise self._unavailable()
return data
except ApiError:
raise
except (ClientError, BotoCoreError, OSError, TimeoutError):
raise self._unavailable() from None
def _delete(self, key: str):
self.client.delete_object(Bucket=self.bucket, Key=key)
async def delete(self, key: str):
try:
await asyncio.to_thread(self._delete, key)
except (ClientError, BotoCoreError, OSError, TimeoutError):
raise self._unavailable() from None
+278
View File
@@ -0,0 +1,278 @@
import uuid
import asyncpg
from app.core.errors import ApiError
from app.repositories.memory import (
AttachmentRow, ChatRow, MessageRow, UserRow, iso_z,
)
def _user(row: asyncpg.Record | None) -> UserRow | None:
if row is None:
return None
return UserRow(
id=str(row['id']), username=row['username'], display_name=row['display_name'],
password_hash=row['password_hash'], created_at=iso_z(row['created_at']),
)
def _chat(row: asyncpg.Record) -> ChatRow:
moment = row['created_at']
return ChatRow(
id=str(row['id']), title=row['title'], member_ids=[str(x) for x in row['member_ids']],
created_dt=moment, created_at=iso_z(moment),
)
def _message(row: asyncpg.Record) -> MessageRow:
moment = row['created_at']
attachment_ids = row['attachment_ids'] if 'attachment_ids' in row else []
return MessageRow(
id=str(row['id']), chat_id=str(row['chat_id']), sender_id=str(row['sender_id']),
text=row['text'], created_dt=moment, created_at=iso_z(moment),
attachment_ids=[str(value) for value in (attachment_ids or [])],
)
def _attachment(row: asyncpg.Record) -> AttachmentRow:
return AttachmentRow(
id=str(row['id']), chat_id=str(row['chat_id']), owner_id=str(row['owner_id']),
object_key=row['object_key'], filename=row['filename'],
content_type=row['content_type'], size_bytes=row['size_bytes'],
sha256=row['sha256'], status=row['status'],
created_at=iso_z(row['created_at']), updated_at=iso_z(row['updated_at']),
)
_MESSAGE_SELECT = (
'SELECT m.*, COALESCE(array_agg(ma.attachment_id ORDER BY ma.position) '
"FILTER (WHERE ma.attachment_id IS NOT NULL), ARRAY[]::uuid[]) AS attachment_ids "
'FROM messages m LEFT JOIN message_attachments ma ON ma.message_id = m.id '
)
class PgStorage:
"""PostgreSQL — источник истины для метаданных; S3 хранит только байты."""
def __init__(self, pool: asyncpg.Pool, session_store, object_store):
self.pool = pool
self.session_store = session_store
self.object_store = object_store
async def ready(self) -> bool:
try:
async with self.pool.acquire() as connection:
database_ready = await connection.fetchval('SELECT 1') == 1
return (
database_ready
and await self.session_store.ready()
and await self.object_store.ready()
)
except (asyncpg.PostgresError, OSError, TimeoutError):
return False
async def create_user(self, username: str, display_name: str, password_hash: str) -> UserRow:
try:
async with self.pool.acquire() as connection:
row = await connection.fetchrow(
'INSERT INTO users (id, username, display_name, password_hash) '
'VALUES ($1, $2, $3, $4) RETURNING *',
uuid.uuid4(), username, display_name, password_hash,
)
except asyncpg.UniqueViolationError as exc:
if exc.constraint_name == 'users_username_key':
raise ApiError(409, 'Username is already taken') from None
raise
return _user(row)
async def user_by_username(self, username: str) -> UserRow | None:
async with self.pool.acquire() as connection:
return _user(await connection.fetchrow(
'SELECT * FROM users WHERE username = $1', username
))
async def user(self, user_id: str) -> UserRow | None:
async with self.pool.acquire() as connection:
return _user(await connection.fetchrow(
'SELECT * FROM users WHERE id = $1', uuid.UUID(user_id)
))
async def create_session(self, user_id: str, ttl_seconds: int):
return await self.session_store.create_session(user_id, ttl_seconds)
async def session(self, token: str):
return await self.session_store.session(token)
async def revoke(self, token: str) -> bool:
return await self.session_store.revoke(token)
async def create_chat(self, title: str, member_ids: list[str], caller_id: str) -> ChatRow:
if caller_id not in member_ids:
raise ApiError(400, 'Creator must be included in member_ids')
chat_id = uuid.uuid4()
try:
async with self.pool.acquire() as connection:
async with connection.transaction():
created_at = await connection.fetchval(
'INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at',
chat_id, title,
)
await connection.executemany(
'INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)',
[(chat_id, uuid.UUID(member_id)) for member_id in member_ids],
)
except asyncpg.ForeignKeyViolationError:
raise ApiError(400, 'Unknown member id') from None
return ChatRow(
id=str(chat_id), title=title, member_ids=list(member_ids),
created_dt=created_at, created_at=iso_z(created_at),
)
async def chat(self, chat_id: str) -> ChatRow | None:
async with self.pool.acquire() as connection:
row = await connection.fetchrow(
'SELECT c.id, c.title, c.created_at, '
'array_agg(cm.user_id ORDER BY cm.user_id) AS member_ids '
'FROM chats c JOIN chat_members cm ON cm.chat_id = c.id '
'WHERE c.id = $1 GROUP BY c.id', uuid.UUID(chat_id),
)
return _chat(row) if row else None
async def chats_of(self, user_id: str) -> list[ChatRow]:
async with self.pool.acquire() as connection:
rows = await connection.fetch(
'SELECT c.id, c.title, c.created_at, '
'array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids '
'FROM chats c '
'JOIN chat_members mine ON mine.chat_id = c.id '
'JOIN chat_members cm_all ON cm_all.chat_id = c.id '
'WHERE mine.user_id = $1 '
'GROUP BY c.id ORDER BY c.created_at, c.id', uuid.UUID(user_id),
)
return [_chat(row) for row in rows]
async def create_attachment(
self, attachment_id: str, chat_id: str, owner_id: str, object_key: str,
filename: str, content_type: str, size_bytes: int, sha256: str,
) -> AttachmentRow:
async with self.pool.acquire() as connection:
row = await connection.fetchrow(
'INSERT INTO attachments '
'(id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256) '
'VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *',
uuid.UUID(attachment_id), uuid.UUID(chat_id), uuid.UUID(owner_id),
object_key, filename, content_type, size_bytes, sha256,
)
return _attachment(row)
async def attachment_for_member(self, attachment_id: str, user_id: str) -> AttachmentRow | None:
try:
attachment_uuid = uuid.UUID(attachment_id)
except ValueError:
raise ApiError(400, 'attachment_id must be a UUID') from None
async with self.pool.acquire() as connection:
row = await connection.fetchrow(
'SELECT a.* FROM attachments a '
'JOIN chat_members cm ON cm.chat_id = a.chat_id '
'WHERE a.id = $1 AND cm.user_id = $2',
attachment_uuid, uuid.UUID(user_id),
)
return _attachment(row) if row else None
async def add_idempotent_message(
self, chat_id: str, sender_id: str, text: str, attachment_ids: list[str],
idempotency_key: str, request_hash: bytes,
) -> MessageRow:
"""Создать сообщение, его вложения и ключ повтора одной транзакцией."""
chat_uuid = uuid.UUID(chat_id)
sender_uuid = uuid.UUID(sender_id)
attachment_uuids = [uuid.UUID(value) for value in attachment_ids]
lock_scope = f'{sender_uuid}:POST:/api/v1/chats/{chat_uuid}/messages:{idempotency_key}'
async with self.pool.acquire() as connection:
async with connection.transaction():
await connection.fetchval(
'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', lock_scope,
)
existing = await connection.fetchrow(
'SELECT k.request_hash, m.id FROM message_idempotency k '
'JOIN messages m ON m.id = k.message_id '
'WHERE k.user_id = $1 AND k.chat_id = $2 AND k.idempotency_key = $3 '
"AND k.created_at > now() - interval '24 hours'",
sender_uuid, chat_uuid, idempotency_key,
)
if existing is not None:
if bytes(existing['request_hash']) != request_hash:
raise ApiError(409, 'Idempotency-Key was already used with another request')
row = await connection.fetchrow(
_MESSAGE_SELECT + 'WHERE m.id = $1 GROUP BY m.id', existing['id'],
)
return _message(row)
if attachment_uuids:
available = await connection.fetch(
"SELECT id FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[]) "
"AND status = 'ready' FOR SHARE",
chat_uuid, attachment_uuids,
)
if len(available) != len(attachment_uuids):
raise ApiError(404, 'Attachment is not available in this chat')
await connection.execute(
'DELETE FROM message_idempotency '
'WHERE user_id = $1 AND chat_id = $2 AND idempotency_key = $3',
sender_uuid, chat_uuid, idempotency_key,
)
await connection.execute(
"DELETE FROM message_idempotency WHERE ctid IN ("
'SELECT ctid FROM message_idempotency '
"WHERE created_at <= now() - interval '24 hours' "
'ORDER BY created_at LIMIT 100)'
)
message_id = uuid.uuid4()
await connection.execute(
'INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4)',
message_id, chat_uuid, sender_uuid, text,
)
if attachment_uuids:
await connection.executemany(
'INSERT INTO message_attachments '
'(message_id, attachment_id, chat_id, position) VALUES ($1, $2, $3, $4)',
[(message_id, attachment_id, chat_uuid, index)
for index, attachment_id in enumerate(attachment_uuids)],
)
row = await connection.fetchrow(
_MESSAGE_SELECT + 'WHERE m.id = $1 GROUP BY m.id', message_id,
)
await connection.execute(
'INSERT INTO message_idempotency '
'(user_id, chat_id, idempotency_key, request_hash, message_id) '
'VALUES ($1, $2, $3, $4, $5)',
sender_uuid, chat_uuid, idempotency_key, request_hash, message_id,
)
return _message(row)
async def messages_page(self, chat_id: str, cursor_key, limit: int):
"""Прочитать ограниченную keyset-страницу вместе с ID прикреплённых файлов."""
chat_uuid = uuid.UUID(chat_id)
base = (
'SELECT m.*, COALESCE(array_agg(ma.attachment_id ORDER BY ma.position) '
"FILTER (WHERE ma.attachment_id IS NOT NULL), ARRAY[]::uuid[]) AS attachment_ids "
'FROM messages m LEFT JOIN message_attachments ma ON ma.message_id = m.id '
'WHERE m.chat_id = $1 '
)
async with self.pool.acquire() as connection:
if cursor_key is None:
rows = await connection.fetch(
base + 'GROUP BY m.id ORDER BY m.created_at, m.id LIMIT $2',
chat_uuid, limit + 1,
)
else:
created_at, id_value = cursor_key
rows = await connection.fetch(
base + 'AND (m.created_at, m.id) > ($2, $3) '
'GROUP BY m.id ORDER BY m.created_at, m.id LIMIT $4',
chat_uuid, created_at, uuid.UUID(int=id_value), limit + 1,
)
has_more = len(rows) > limit
return [_message(row) for row in rows[:limit]], has_more
+66
View File
@@ -0,0 +1,66 @@
import hashlib
import time
from datetime import datetime, timedelta, timezone
from redis.asyncio import Redis
from redis.exceptions import RedisError
from app.core.errors import ApiError
from app.core.security import new_token
from app.repositories.memory import SessionRow, now_utc
class RedisSessionStore:
"""Redis — источник истины для сессий; PXAT сохраняет исходный срок истечения после перезапуска."""
def __init__(self, client: Redis):
self.client = client
@staticmethod
def _key(token: str) -> str:
digest = hashlib.sha256(token.encode('utf-8')).hexdigest()
return 'session:' + digest
async def ready(self) -> bool:
try:
return bool(await self.client.ping())
except (RedisError, OSError, TimeoutError):
return False
async def create_session(self, user_id: str, ttl_seconds: int) -> SessionRow:
token = new_token()
current = now_utc()
expires_at = current.replace(microsecond=(current.microsecond // 1000) * 1000) + timedelta(seconds=ttl_seconds)
expires_ms = int(expires_at.timestamp() * 1000)
try:
await self.client.set(
self._key(token), f'{user_id}|{expires_ms}', pxat=expires_ms
)
except RedisError:
raise ApiError(503, 'Session storage is unavailable') from None
return SessionRow(token=token, user_id=user_id, expires_at=expires_at)
async def session(self, token: str) -> SessionRow | None:
try:
value = await self.client.get(self._key(token))
except RedisError:
raise ApiError(503, 'Session storage is unavailable') from None
if value is None:
return None
try:
user_id, expires_ms_text = value.split('|', 1)
expires_ms = int(expires_ms_text)
if expires_ms <= int(time.time() * 1000):
await self.client.delete(self._key(token))
return None
expires_at = datetime.fromtimestamp(expires_ms / 1000, timezone.utc)
return SessionRow(token=token, user_id=user_id, expires_at=expires_at)
except (ValueError, OverflowError):
await self.client.delete(self._key(token))
return None
async def revoke(self, token: str) -> bool:
try:
return bool(await self.client.delete(self._key(token)))
except RedisError:
raise ApiError(503, 'Session storage is unavailable') from None
+85
View File
@@ -0,0 +1,85 @@
import uuid as _uuid
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
STRICT = ConfigDict(extra='forbid')
class CreateUser(BaseModel):
model_config = STRICT
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator('display_name', 'password')
@classmethod
def _not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError('string must not consist of whitespace only')
return value
class User(BaseModel):
model_config = STRICT
id: _uuid.UUID
username: str
display_name: str
created_at: str
class Session(BaseModel):
model_config = STRICT
token: str
token_type: Literal['Bearer']
expires_at: str
user: User
class CreateChat(BaseModel):
model_config = STRICT
title: str = Field(min_length=1, max_length=100)
member_ids: list[_uuid.UUID] = Field(min_length=1, max_length=100)
@field_validator('title')
@classmethod
def _title_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError('string must not consist of whitespace only')
return value
@field_validator('member_ids')
@classmethod
def _unique_members(cls, value):
if len(set(value)) != len(value):
raise ValueError('duplicate member ids')
return value
class CreateMessage(BaseModel):
model_config = STRICT
text: str = Field(min_length=0, max_length=4000)
attachment_ids: list[_uuid.UUID] = Field(default_factory=list, max_length=10)
@field_validator('attachment_ids')
@classmethod
def _unique_attachments(cls, value):
if len(set(value)) != len(value):
raise ValueError('duplicate attachment ids')
return value
class Attachment(BaseModel):
model_config = STRICT
id: _uuid.UUID
chat_id: _uuid.UUID
owner_id: _uuid.UUID
filename: str = Field(min_length=1, max_length=255)
content_type: Literal['image/png', 'image/jpeg']
size_bytes: int = Field(ge=1, le=10_485_760)
sha256: str = Field(pattern=r'^[a-f0-9]{64}$')
status: Literal['ready']
variants: list = Field(default_factory=list)
error_code: str | None = None
created_at: str
updated_at: str
+130
View File
@@ -0,0 +1,130 @@
name: mtusi-lab4
services:
postgres:
image: postgres:16.4-alpine
environment:
POSTGRES_USER: ${POSTGRES_USER:?Set POSTGRES_USER in .env}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
POSTGRES_DB: ${POSTGRES_DB:?Set POSTGRES_DB in .env}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
interval: 3s
timeout: 3s
retries: 30
start_period: 5s
restart: unless-stopped
redis:
image: redis:7.4.2-alpine
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", ""]
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 3s
timeout: 3s
retries: 30
start_period: 5s
restart: unless-stopped
s3:
image: ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d
command: ["/data"]
environment:
RUSTFS_VOLUMES: /data
RUSTFS_ADDRESS: 0.0.0.0:9000
RUSTFS_CONSOLE_ENABLE: "false"
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
RUSTFS_SECRET_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
RUSTFS_OBS_LOGGER_LEVEL: error
volumes:
- s3_data:/data
restart: unless-stopped
migrate:
build:
context: .
dockerfile: Dockerfile
command: ["python", "-m", "app.db.migrate"]
environment:
DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
DB_POOL_MIN_SIZE: 1
DB_POOL_MAX_SIZE: 2
DB_POOL_ACQUIRE_TIMEOUT_SECONDS: 3
DB_COMMAND_TIMEOUT_SECONDS: 10
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build:
context: .
dockerfile: Dockerfile
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
DB_POOL_MIN_SIZE: ${DB_POOL_MIN_SIZE:-1}
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
DB_POOL_ACQUIRE_TIMEOUT_SECONDS: ${DB_POOL_ACQUIRE_TIMEOUT_SECONDS:-3}
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-5}
REDIS_URL: redis://redis:6379/0
REDIS_TIMEOUT_SECONDS: ${REDIS_TIMEOUT_SECONDS:-2}
S3_ENDPOINT_URL: http://s3:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-3}
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-10}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
depends_on:
migrate:
condition: service_completed_successfully
postgres:
condition: service_healthy
redis:
condition: service_healthy
s3:
condition: service_started
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 3s
timeout: 3s
retries: 20
start_period: 10s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "1.0"
memory: 512M
stop_grace_period: 20s
restart: unless-stopped
lb:
image: nginx:1.27.4-alpine
ports:
- "127.0.0.1:${LB_PORT:-8080}:8080"
volumes:
- ./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
interval: 3s
timeout: 3s
retries: 10
restart: unless-stopped
volumes:
postgres_data:
redis_data:
s3_data:
+1
View File
@@ -0,0 +1 @@
PASS: docker compose config --quiet
+7
View File
@@ -0,0 +1,7 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 4 minutes ago Up 3 minutes (healthy) 8080/tcp
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 4 minutes ago Up 4 minutes (healthy) 8080/tcp
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 4 minutes ago Up 4 minutes (healthy) 127.0.0.1:8080->8080/tcp
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 4 minutes ago Up 4 minutes (healthy) 5432/tcp
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 4 minutes ago Up 4 minutes (healthy) 6379/tcp
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 4 minutes ago Up 4 minutes 9000-9001/tcp
+1
View File
@@ -0,0 +1 @@
PASS: compose.yaml parses
+1
View File
@@ -0,0 +1 @@
PASS: final git diff --check
+8
View File
@@ -0,0 +1,8 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 26 seconds ago Up 19 seconds (healthy) 8080/tcp
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 26 seconds ago Up 19 seconds (healthy) 8080/tcp
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 26 seconds ago Up 13 seconds (healthy) 127.0.0.1:8080->8080/tcp
mtusi-lab4-migrate-1 mtusi-lab4-migrate "python -m app.db.mi…" migrate 26 seconds ago Exited (0) 19 seconds ago
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 26 seconds ago Up 25 seconds (healthy) 5432/tcp
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 26 seconds ago Up 25 seconds (healthy) 6379/tcp
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 26 seconds ago Up 25 seconds 9000-9001/tcp
+1
View File
@@ -0,0 +1 @@
+8
View File
@@ -0,0 +1,8 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 49 seconds ago Up 42 seconds (healthy) 8080/tcp
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 49 seconds ago Up 5 seconds (healthy) 8080/tcp
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 49 seconds ago Up 36 seconds (healthy) 127.0.0.1:8080->8080/tcp
mtusi-lab4-migrate-1 mtusi-lab4-migrate "python -m app.db.mi…" migrate 49 seconds ago Exited (0) 5 seconds ago
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 49 seconds ago Up 48 seconds (healthy) 5432/tcp
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 49 seconds ago Up 48 seconds (healthy) 6379/tcp
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 49 seconds ago Up 48 seconds 9000-9001/tcp
+8
View File
@@ -0,0 +1,8 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 42 seconds ago Up 35 seconds (healthy) 8080/tcp
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 42 seconds ago Exited (0) 14 seconds ago
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 42 seconds ago Up 29 seconds (healthy) 127.0.0.1:8080->8080/tcp
mtusi-lab4-migrate-1 mtusi-lab4-migrate "python -m app.db.mi…" migrate 42 seconds ago Exited (0) 35 seconds ago
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 42 seconds ago Up 41 seconds (healthy) 5432/tcp
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 42 seconds ago Up 41 seconds (healthy) 6379/tcp
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 42 seconds ago Up 41 seconds 9000-9001/tcp
+1
View File
@@ -0,0 +1 @@
PASS: restart/failover hooks syntax
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+6
View File
@@ -0,0 +1,6 @@
python3 scripts/validate_openapi.py
Traceback (most recent call last):
File "/home/lenovo/lab4/mtusi2026lab4-fork/scripts/validate_openapi.py", line 5, in <module>
from openapi_spec_validator import validate
ModuleNotFoundError: No module named 'openapi_spec_validator'
make: *** [Makefile:16: validate] Error 1
+2
View File
@@ -0,0 +1,2 @@
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+15
View File
@@ -0,0 +1,15 @@
Container mtusi-lab4-postgres-1 Running
Container mtusi-lab4-postgres-1 Waiting
Container mtusi-lab4-postgres-1 Healthy
Container mtusi-lab4-migrate-run-3a566e7f2cbd Creating
Container mtusi-lab4-migrate-run-3a566e7f2cbd Created
Container mtusi-lab4-postgres-1 Running
Container mtusi-lab4-postgres-1 Waiting
Container mtusi-lab4-postgres-1 Healthy
Container mtusi-lab4-migrate-run-5e9470435d61 Creating
Container mtusi-lab4-migrate-run-5e9470435d61 Created
Migration versions:
1
2
3
4
+7
View File
@@ -0,0 +1,7 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 12 seconds ago Up 6 seconds (healthy) 8080/tcp
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 12 seconds ago Up 5 seconds (healthy) 8080/tcp
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 12 seconds ago Up Less than a second (health: starting) 127.0.0.1:8080->8080/tcp
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 12 seconds ago Up 11 seconds (healthy) 5432/tcp
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 12 seconds ago Up 11 seconds (healthy) 6379/tcp
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 12 seconds ago Up 12 seconds 9000-9001/tcp
+7
View File
@@ -0,0 +1,7 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 2 minutes ago Up About a minute (healthy) 8080/tcp
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:8080->8080/tcp
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 2 minutes ago Up 2 minutes (healthy) 5432/tcp
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 2 minutes ago Up 2 minutes (healthy) 6379/tcp
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 2 minutes ago Up 2 minutes 9000-9001/tcp
+1
View File
@@ -0,0 +1 @@
PASS: compileall app student-tests scripts
+1
View File
@@ -0,0 +1 @@
+21
View File
@@ -0,0 +1,21 @@
Container mtusi-lab4-postgres-1 Running
Container mtusi-lab4-redis-1 Running
Container mtusi-lab4-s3-1 Running
Container mtusi-lab4-api-2 Running
Container mtusi-lab4-lb-1 Running
Container mtusi-lab4-postgres-1 Waiting
Container mtusi-lab4-postgres-1 Healthy
Container mtusi-lab4-migrate-1 Starting
Container mtusi-lab4-migrate-1 Started
Container mtusi-lab4-migrate-1 Waiting
Container mtusi-lab4-postgres-1 Waiting
Container mtusi-lab4-redis-1 Waiting
Container mtusi-lab4-postgres-1 Healthy
Container mtusi-lab4-redis-1 Healthy
Container mtusi-lab4-migrate-1 Exited
Container mtusi-lab4-api-1 Starting
Container mtusi-lab4-api-1 Started
Container mtusi-lab4-api-1 Waiting
Container mtusi-lab4-api-2 Waiting
Container mtusi-lab4-api-1 Healthy
Container mtusi-lab4-api-2 Healthy
+7
View File
@@ -0,0 +1,7 @@
Static checks (2026-11-02; base commit recorded in verification-metadata.txt)
make-check: PASS
make-validate: PASS
python-compileall: PASS
hooks-syntax: PASS
compose-yaml: PASS
diff-check: PASS
+2
View File
@@ -0,0 +1,2 @@
python3 student-tests/attachment_failover.py --base-url "http://localhost:8080"
PASS: upload instance stopped; metadata and original bytes read via peer
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 140, "failed_attempts": 0, "first_recovery_seconds": 0.556}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
python3 student-tests/idempotency.py --base-url "http://localhost:8080"
PASS: 16 concurrent requests across 2 API replicas returned one Message; changed body returned 409.
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+3
View File
@@ -0,0 +1,3 @@
python3 student-tests/private_bucket.py
HTTP 403
PASS: private bucket rejects unauthenticated listing
+7
View File
@@ -0,0 +1,7 @@
# Анализ первого запуска S3 outage-теста
Дата: 2026-11-02.
Первый запуск остановил S3 и успел создать и прочитать синтетическое вложение. Проверочный клиент имел timeout=5 s и прервал GET /health/ready раньше, чем завершились ограниченные connect/read/retry таймауты S3 SDK. Сценарий восстановления в finally запустил S3 обратно; readiness снова стала 200, исходные байты остались доступны.
В тесте увеличен только клиентский timeout до 35 s, чтобы наблюдать фактический HTTP-ответ после bounded retries. Настройки production S3 не менялись. Повторный прогон получил 503 на readiness и скачивание во время отказа, сохранил доступ к metadata PostgreSQL и подтвердил чтение оригинальных байтов после восстановления S3. Итоговый stdout: test-s3-failure.txt; первоначальный stdout: test-s3-failure-first-attempt.txt.
@@ -0,0 +1,3 @@
python3 student-tests/s3_failure.py --base-url "http://localhost:8080"
FAIL: GET /health/ready: connection/TLS failure (TimeoutError)
make: *** [Makefile:31: test-s3-failure] Error 1
+3
View File
@@ -0,0 +1,3 @@
python3 student-tests/s3_failure.py --base-url "http://localhost:8080"
PASS: S3 outage returned 503 for readiness/download; metadata remained available
PASS: original attachment bytes were readable after S3 recovery
+2
View File
@@ -0,0 +1,2 @@
python3 student-tests/attachments.py --base-url "http://localhost:8080"
PASS: upload/checksum/download, chat ACL, message binding and upload limits
+12
View File
@@ -0,0 +1,12 @@
Date: 2026-11-02
Base commit: ec540a1
Branch: solution/BVT2302-Fedotov
Working tree: uncommitted at author's request; no commit or push created
Docker Engine: 29.8.0
WSL kernel: 6.18.33.2-microsoft-standard-WSL2
CPU threads: 32
Memory available to WSL: 15 GiB
Compose limits: each API replica 1 CPU / 512 MiB; other containers have no explicit resource limits
Verification profile: make check; full OpenAPI validation; public smoke; upload/download, checksum, ACL and upload limits; 16-request idempotency race across two APIs; private bucket anonymous listing; S3 outage and recovery; cross-replica attachment read; full persistence restart; 140-write API failover; two repeated migration runs (versions 1–4).
Load benchmark: not run.
Evidence contents: small sanitized command/stdout summaries only; no .env, credentials, tokens, user payloads, or raw dumps.
+37
View File
@@ -0,0 +1,37 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (length(title) BETWEEN 1 AND 100 AND btrim(title) <> ''),
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL CHECK (length(text) BETWEEN 1 AND 4000 AND btrim(text) <> ''),
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
-- A leaked session table must not contain usable bearer credentials.
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
@@ -0,0 +1,3 @@
-- Active sessions move to Redis in lab2 grade 4; old token digests cannot be
-- reconstructed and are intentionally invalidated during this one-time migration.
DROP TABLE IF EXISTS sessions;
+10
View File
@@ -0,0 +1,10 @@
CREATE TABLE message_idempotency (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
idempotency_key text NOT NULL CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
request_hash bytea NOT NULL CHECK (octet_length(request_hash) = 32),
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, idempotency_key)
);
CREATE INDEX message_idempotency_expiry_idx ON message_idempotency (created_at);
+37
View File
@@ -0,0 +1,37 @@
ALTER TABLE messages
DROP CONSTRAINT messages_text_check,
ADD CONSTRAINT messages_text_check CHECK (length(text) BETWEEN 0 AND 4000);
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id),
object_key text NOT NULL UNIQUE,
filename text NOT NULL CHECK (length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
status text NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (variants = '[]'::jsonb),
error_code text,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE (id, chat_id)
);
CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id);
ALTER TABLE messages ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
attachment_id uuid NOT NULL,
chat_id uuid NOT NULL,
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
UNIQUE (message_id, position),
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE,
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) ON DELETE RESTRICT
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
+29
View File
@@ -0,0 +1,29 @@
# DNS в Compose обновляется после замены остановленной API-реплики.
resolver 127.0.0.11 valid=1s ipv6=off;
upstream api_backend {
zone api_backend 64k;
server api:8080 resolve max_fails=1 fail_timeout=2s;
keepalive 32;
}
server {
listen 8080;
server_name _;
client_max_body_size 12m;
location / {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 2s;
proxy_send_timeout 30s;
proxy_read_timeout 30s;
proxy_request_buffering off;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
}
}
+8
View File
@@ -0,0 +1,8 @@
fastapi==0.115.6
uvicorn==0.34.0
pydantic-settings==2.7.1
argon2-cffi==23.1.0
asyncpg==0.30.0
redis==5.2.1
boto3==1.36.26
Pillow==11.1.0
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
# compose down удаляет контейнеры и сеть, но оставляет именованные volumes.
docker compose --env-file .env down
docker compose --env-file .env up -d
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides X-Instance-Id of the target replica}"
for container in $(docker compose ps -q api); do
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container")
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
# Manual stop leaves this replica out of service until the operator restores it.
docker stop --timeout 1 "$container" >/dev/null
exit 0
fi
done
printf '%s\n' 'target API replica was not found; no container was stopped' >&2
exit 1
+31
View File
@@ -0,0 +1,31 @@
# Собственные проверки Lab 4
Этот каталог принадлежит студенту. Публичные contract tests в `tests/` сохранены без изменений.
Запуск после старта стенда:
```sh
make test-student BASE_URL=http://localhost:8080
```
Проверка создаёт синтетических пользователей и чат, загружает публичный PNG fixture, читает его как второй участник, проверяет ACL для постороннего, привязку вложения к чату и типовые ошибки upload. Она не печатает токены и не сохраняет пользовательские данные.
Для обязательного эксперимента чтения между API-репликами выполните после smoke:
```sh
make test-attachment-failover BASE_URL=http://localhost:8080
```
Скрипт останавливает только API-реплику, которая обслужила upload, читает оригинал через LB, затем восстанавливает две реплики. Все хранилища и volumes остаются на месте.
Проверка приватности bucket из сети API:
```sh
make test-private-bucket
```
Проверка недоступности и восстановления S3 (временно останавливает только сервис s3, затем запускает его обратно):
```sh
make test-s3-failure BASE_URL=http://localhost:8080
```
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Upload through one API instance, stop it, then download through its peer."""
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, arguments, check, run
def main():
args = arguments('Lab 4 attachment cross-replica check')
client = Client(args.base_url, args.ca_file, timeout=3)
alice, bob, eve, chat = client.bootstrap()
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
uploaded = client.call(
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
201, actor=alice, raw=content, headers={'Content-Type': 'image/png'},
schema='Attachment',
)
source_instance = uploaded['headers']['X-Instance-Id']
observed = set()
for _ in range(50):
observed.add(client.call(
'GET', '/health/live', schema='Health',
)['headers']['X-Instance-Id'])
check(len(observed) >= 2, 'expected at least two API instances before stopping one')
check(source_instance in observed, 'upload instance was not observed by the load balancer')
hook = ROOT / 'scripts/stop-one.sh'
env = dict(os.environ)
env['TARGET_INSTANCE_ID'] = source_instance
subprocess_result = subprocess.run(
[str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=10, check=False,
)
check(subprocess_result.returncode == 0, 'stop-one hook could not stop the upload instance')
try:
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
try:
metadata = client.call(
'GET', '/api/v1/attachments/' + uploaded['json']['id'],
actor=bob, schema='Attachment',
)
download = client.call(
'GET', '/api/v1/attachments/' + uploaded['json']['id'] + '/content',
actor=bob,
)
if (metadata['headers']['X-Instance-Id'] != source_instance
and download['headers']['X-Instance-Id'] != source_instance):
check(download['bytes'] == content, 'cross-replica bytes differ from uploaded file')
check(metadata['json']['sha256'] == uploaded['json']['sha256'],
'cross-replica metadata checksum changed')
print('PASS: upload instance stopped; metadata and original bytes read via peer')
return
except Failure:
time.sleep(0.25)
raise Failure('attachment was not readable from the other API instance within 10 seconds')
finally:
# Restore the two-replica local test stack after this explicit failure experiment.
subprocess.run(
['docker', 'compose', '--env-file', '.env', 'up', '-d', '--scale', 'api=2'],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
timeout=60, check=False,
)
if __name__ == '__main__':
run(main)
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""Собственные интеграционные проверки вложений поверх неизменённого публичного API."""
import hashlib
from pathlib import Path
import sys
from urllib.parse import quote
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments('Lab 4 student attachment checks')
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
fixture = (ROOT / 'tests/fixtures/sample.png').read_bytes()
uploaded = client.call(
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
201, actor=alice, raw=fixture, headers={'Content-Type': 'image/png'},
schema='Attachment',
)['json']
check(uploaded['status'] == 'ready', 'upload was not ready')
check(uploaded['variants'] == [] and uploaded['error_code'] is None,
'Lab 4 ready metadata must have no variants or error')
check(uploaded['sha256'] == hashlib.sha256(fixture).hexdigest(),
'metadata checksum differs from fixture')
check(uploaded['size_bytes'] == len(fixture), 'metadata size differs from fixture')
metadata = client.call(
'GET', '/api/v1/attachments/' + uploaded['id'],
actor=bob, schema='Attachment',
)['json']
check(metadata == uploaded, 'another chat participant got different attachment metadata')
downloaded = client.call(
'GET', '/api/v1/attachments/' + uploaded['id'] + '/content', actor=bob,
)
check(downloaded['bytes'] == fixture, 'downloaded bytes are not identical')
check(downloaded['headers'].get_content_type() == 'image/png',
'download Content-Type is not the detected image type')
client.call('GET', '/api/v1/attachments/' + uploaded['id'], 404, actor=eve)
client.call('GET', '/api/v1/attachments/' + uploaded['id'] + '/content', 404, actor=eve)
# Нельзя привязать существующий файл чата к сообщению другого чата.
other_chat = client.call(
'POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]},
schema='Chat',
)['json']
client.call(
'POST', '/api/v1/chats/' + other_chat['id'] + '/messages', 404,
actor=alice, body={'text': '', 'attachment_ids': [uploaded['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex},
)
prefix = '/api/v1/chats/' + chat['id'] + '/attachments?filename='
client.call(
'POST', prefix + 'bad.png', 415, actor=alice, raw=b'not an image',
headers={'Content-Type': 'image/png'},
)
client.call(
'POST', prefix + 'mismatch.jpg', 415, actor=alice, raw=fixture,
headers={'Content-Type': 'image/jpeg'},
)
client.call(
'POST', prefix + quote('../sample.png', safe=''), 400, actor=alice,
raw=fixture, headers={'Content-Type': 'image/png'},
)
client.call(
'POST', prefix + 'too-large.png', 413, actor=alice,
raw=b'\x00' * (10 * 1024 * 1024 + 1),
headers={'Content-Type': 'image/png'},
)
print('PASS: upload/checksum/download, chat ACL, message binding and upload limits')
if __name__ == '__main__':
run(main)
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env python3
"""Проверка атомарного повтора одного сообщения через разные API-реплики."""
from concurrent.futures import ThreadPoolExecutor
import uuid
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, arguments, check, run
def main():
args = arguments('Concurrent cross-replica idempotency check')
setup = Client(args.base_url, args.ca_file)
alice, _bob, _eve, chat = setup.bootstrap()
key = 'concurrent-' + uuid.uuid4().hex
text = 'Одно сообщение при конкурентном повторе ' + uuid.uuid4().hex
count = 16
def send(_):
client = Client(args.base_url, args.ca_file, timeout=10)
return client.call(
'POST', '/api/v1/chats/' + chat['id'] + '/messages', expected=201,
actor=alice, body={'text': text}, headers={'Idempotency-Key': key}, schema='Message',
)
with ThreadPoolExecutor(max_workers=count) as pool:
responses = list(pool.map(send, range(count)))
messages = [response['json'] for response in responses]
instance_ids = {response['headers'].get('X-Instance-Id') for response in responses}
check(all(message == messages[0] for message in messages), 'concurrent replay returned different Message objects')
check(len(instance_ids) >= 2, 'concurrent requests did not reach at least two observable API replicas')
replay = setup.call(
'POST', '/api/v1/chats/' + chat['id'] + '/messages', expected=201,
actor=alice, body={'text': text}, headers={'Idempotency-Key': key}, schema='Message',
)['json']
check(replay == messages[0], 'retry returned a different status payload')
setup.call(
'POST', '/api/v1/chats/' + chat['id'] + '/messages', expected=409,
actor=alice, body={'text': text + ' другой текст'}, headers={'Idempotency-Key': key},
)
history = setup.call(
'GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100',
actor=alice, schema='MessagePage',
)['json']['items']
matching = [message for message in history if message['text'] == text]
check(matching == [messages[0]], 'history contains a missing or duplicate side effect')
print('PASS: %d concurrent requests across %d API replicas returned one Message; changed body returned 409.' % (count, len(instance_ids)))
if __name__ == '__main__':
run(main)
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""Отправить анонимный ListBucket из сети приложения и подтвердить private bucket."""
from pathlib import Path
import subprocess
ROOT = Path(__file__).resolve().parents[1]
def main():
env_path = ROOT / '.env'
if not env_path.is_file():
raise SystemExit('Create local .env from .env.example first')
values = {}
for line in env_path.read_text().splitlines():
if '=' in line and not line.lstrip().startswith('#'):
key, value = line.split('=', 1)
values[key.strip()] = value.strip()
bucket = values.get('S3_BUCKET', 'messenger-attachments')
url = 'http://s3:9000/' + bucket + '?list-type=2'
code = (
'import urllib.error, urllib.request\n'
'url = ' + repr(url) + '\n'
'try:\n'
' urllib.request.urlopen(url, timeout=5)\n'
'except urllib.error.HTTPError as error:\n'
' print("HTTP", error.code)\n'
' raise SystemExit(0 if error.code in (401, 403) else 1)\n'
'except Exception as error:\n'
' print(type(error).__name__)\n'
' raise SystemExit(1)\n'
'raise SystemExit("anonymous bucket listing unexpectedly succeeded")\n'
)
result = subprocess.run(
['docker', 'compose', '--env-file', '.env', 'exec', '-T', 'api', 'python', '-c', code],
cwd=ROOT, check=False,
)
if result.returncode:
raise SystemExit('FAIL: anonymous S3 bucket listing was not denied')
print('PASS: private bucket rejects unauthenticated listing')
if __name__ == '__main__':
main()
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
"""Проверить ограниченную ошибку API при отказе S3 и восстановить его."""
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, arguments, check, run
def main():
args = arguments('Lab 4 S3 outage and recovery check')
# Дождаться ограниченных повторов boto3 при остановленном S3.
client = Client(args.base_url, args.ca_file, timeout=35)
alice, bob, eve, chat = client.bootstrap()
attachment, content = client.attachment(alice, chat)
initial = client.call(
'GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob,
)
check(initial['bytes'] == content, 'uploaded bytes differ before the outage')
stopped = subprocess.run(
['docker', 'compose', '--env-file', '.env', 'stop', 's3'],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
timeout=20, check=False,
)
check(stopped.returncode == 0, 'could not stop S3 for the outage test')
try:
ready = client.call('GET', '/health/ready', expected=503, schema='Error')
check(ready['json']['error']['code'] == 'unavailable',
'readiness did not report unavailable S3')
failed = client.call(
'GET', '/api/v1/attachments/' + attachment['id'] + '/content',
expected=503, actor=bob, schema='Error',
)
check(failed['json']['error']['code'] == 'unavailable',
'S3 failure was not translated to a bounded API error')
metadata = client.call(
'GET', '/api/v1/attachments/' + attachment['id'],
actor=bob, schema='Attachment',
)
check(metadata['json']['id'] == attachment['id'],
'PostgreSQL metadata was lost while S3 was unavailable')
print('PASS: S3 outage returned 503 for readiness/download; metadata remained available')
finally:
started = subprocess.run(
['docker', 'compose', '--env-file', '.env', 'start', 's3'],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
timeout=20, check=False,
)
check(started.returncode == 0, 'could not restart S3 after the outage test')
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
try:
client.call('GET', '/health/ready', schema='Health')
break
except Failure:
time.sleep(0.25)
else:
raise Failure('API did not become ready after S3 restart')
recovered = client.call(
'GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob,
)
check(recovered['bytes'] == content, 'S3 restart changed or lost the original bytes')
print('PASS: original attachment bytes were readable after S3 recovery')
if __name__ == '__main__':
run(main)