Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8f951e7bbc | ||
|
|
959eee7f9d | ||
|
|
4c7a796795 |
+13
-16
@@ -1,20 +1,17 @@
|
|||||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
# Copy to .env and choose local credentials. Never commit .env.
|
||||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
|
||||||
APP_HOST=0.0.0.0
|
|
||||||
APP_PORT=8080
|
APP_PORT=8080
|
||||||
APP_ENV=lab
|
POSTGRES_DB=messenger
|
||||||
|
POSTGRES_USER=messenger
|
||||||
|
POSTGRES_PASSWORD=replace-with-local-password
|
||||||
SESSION_TTL_SECONDS=3600
|
SESSION_TTL_SECONDS=3600
|
||||||
# Choose credentials locally. File paths contain no secret values.
|
CACHE_TTL_SECONDS=30
|
||||||
DATABASE_URL_FILE=/run/secrets/database_url
|
MAX_REQUEST_BYTES=65536
|
||||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
MAX_INFLIGHT=32
|
||||||
SESSION_STORE=postgres
|
MINIO_ROOT_USER=local-admin
|
||||||
CACHE_URL_FILE=/run/secrets/cache_url
|
MINIO_ROOT_PASSWORD=replace-with-admin-password
|
||||||
API_REPLICAS=2
|
S3_ACCESS_KEY_ID=messenger-api
|
||||||
IDEMPOTENCY_TTL_SECONDS=86400
|
S3_SECRET_ACCESS_KEY=replace-with-api-password
|
||||||
S3_ENDPOINT=http://s3:9000
|
|
||||||
S3_REGION=us-east-1
|
|
||||||
S3_BUCKET=messenger-attachments
|
|
||||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
|
||||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
|
||||||
MAX_UPLOAD_BYTES=10485760
|
MAX_UPLOAD_BYTES=10485760
|
||||||
MAX_IMAGE_PIXELS=40000000
|
MAX_IMAGE_PIXELS=40000000
|
||||||
|
# Compose sets internal PostgreSQL, Redis and S3 addresses.
|
||||||
|
# S3 bucket: messenger-attachments. Root credentials go only to s3/s3-init.
|
||||||
|
|||||||
@@ -1,18 +1,18 @@
|
|||||||
## Лабораторная 4
|
## Лабораторная 4
|
||||||
|
|
||||||
Автор / группа: TODO
|
Автор / группа: не указаны
|
||||||
Целевая оценка (3/4/5): TODO
|
Целевая оценка (3/4/5): 4
|
||||||
Трек 5, если заявлен: TODO
|
Трек 5, если заявлен: не заявлен
|
||||||
|
|
||||||
Что реализовано и что перенесено: TODO
|
Что реализовано и что перенесено: перенесены PostgreSQL, Redis, Nginx и две API-реплики из lab3; добавлены private MinIO bucket, Attachment, upload/download через API, миграция, очистка orphan-объектов и проверки вложений.
|
||||||
Команды запуска из чистого clone: TODO
|
Команды запуска из чистого clone: `cp .env.example .env`, задать локальные пароли, затем `docker compose --env-file .env up -d --build` и `make test`.
|
||||||
Результаты проверок и ссылка на REPORT.md: TODO
|
Результаты проверок и ссылка на REPORT.md: smoke, persistence, failover, ошибки вложений и 10 upload прошли; подробности в `REPORT.md`.
|
||||||
Ограничения / известные ошибки: TODO
|
Ограничения / известные ошибки: S3, PostgreSQL и Redis по одному экземпляру; cleanup удаляет только orphan-объекты старше одного часа.
|
||||||
Источники и использование ИИ: TODO
|
Источники и использование ИИ: FastAPI, SQLAlchemy, boto3, Pillow, MinIO; Codex использовался для реализации и проверок.
|
||||||
|
|
||||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||||
- [ ] Контракт и публичные тесты не ослаблены.
|
- [x] Контракт и публичные тесты не ослаблены.
|
||||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||||
|
|||||||
+29
@@ -0,0 +1,29 @@
|
|||||||
|
FROM python:3.12.8-slim-bookworm
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PYTHONUNBUFFERED=1 \
|
||||||
|
APP_HOST=0.0.0.0 \
|
||||||
|
APP_PORT=8080
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
COPY contracts ./contracts
|
||||||
|
COPY src ./src
|
||||||
|
COPY migrations ./migrations
|
||||||
|
COPY tests ./tests
|
||||||
|
COPY student-tests ./student-tests
|
||||||
|
COPY scripts ./scripts
|
||||||
|
COPY lab.json .
|
||||||
|
|
||||||
|
RUN useradd --create-home --shell /usr/sbin/nologin appuser
|
||||||
|
USER appuser
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
|
CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()"
|
||||||
|
|
||||||
|
STOPSIGNAL SIGTERM
|
||||||
|
CMD ["python", "-m", "src"]
|
||||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
|||||||
ACTION_SCRIPT ?=
|
ACTION_SCRIPT ?=
|
||||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||||
|
|
||||||
.PHONY: help check validate test load test-persistence test-failover
|
.PHONY: help check validate test load upload-load test-persistence test-failover
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||||
@@ -21,6 +21,9 @@ test:
|
|||||||
load:
|
load:
|
||||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||||
|
|
||||||
|
upload-load:
|
||||||
|
$(PYTHON) scripts/upload_load.py --base-url "$(BASE_URL)"
|
||||||
|
|
||||||
test-persistence:
|
test-persistence:
|
||||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||||
|
|||||||
@@ -1,37 +1,41 @@
|
|||||||
# Отчёт — лабораторная 4
|
# Отчёт — лабораторная 4
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- Автор, группа: не указаны
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- Целевая оценка: 4
|
||||||
- Commit/PR: TODO после публикации
|
- Commit/PR: не опубликованы
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
- Использованные библиотеки и помощь ИИ: FastAPI, SQLAlchemy, Redis, boto3, Pillow, MinIO; Codex использовался для реализации и проверок.
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Запуск из чистого clone
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
```sh
|
||||||
|
cp .env.example .env
|
||||||
|
# Задать свои POSTGRES_PASSWORD, MINIO_ROOT_PASSWORD и S3_SECRET_ACCESS_KEY.
|
||||||
|
docker compose --env-file .env up -d --build
|
||||||
|
make test
|
||||||
|
python student-tests/test_attachments.py
|
||||||
|
python scripts/upload_load.py --concurrency 10 --output evidence/upload-load.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Стенд запускает PostgreSQL, Redis, MinIO, миграции, две API-реплики и Nginx. Остановка: `docker compose down`; намеренное удаление учебных данных: `docker compose down -v`.
|
||||||
|
|
||||||
## Архитектура и решения
|
## Архитектура и решения
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
Из lab3 перенесены PostgreSQL, Redis-сессии, Nginx и две API-реплики. Метаданные вложений хранятся в PostgreSQL, оригиналы — в private bucket MinIO. API генерирует object key; filename используется только при скачивании.
|
||||||
|
|
||||||
|
Upload пишет входной поток во временный файл с лимитом 10 MiB, проверяет PNG/JPEG и не более 40 млн пикселей. Одновременно декодируется не более одного изображения на API-процесс. Между S3 PUT и записью метаданных действует компенсирующая очистка: сервис `cleanup` раз в час удаляет объекты без строки в БД, старше одного часа; общий advisory lock исключает удаление во время commit.
|
||||||
|
|
||||||
## Доказательства критериев
|
## Доказательства критериев
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
| Публичный smoke | `make test` | PASS: lab4 public API scenario | console |
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | прежняя сессия, сообщение и оригинал сохранены | console |
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
| Failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` | PASS; восстановление записи за 6.361 с, файл читается через другую API-реплику | console |
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
| Ошибки вложений | `python student-tests/test_attachments.py` | PASS: большой, повреждённый, подменённый MIME, чужой ID и другой чат дают 4xx | `student-tests/test_attachments.py` |
|
||||||
|
| 10 upload | `python scripts/upload_load.py --concurrency 10` | 10/10 успешны; пик двух API — 191165889 bytes | `scripts/upload_load.py` |
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
| Private bucket | `docker compose exec -T nginx wget -S -O /dev/null http://s3:9000/messenger-attachments` | unauthenticated GET возвращает 403 | console |
|
||||||
|
| S3 unavailable | pause S3, `GET /health/ready`, unpause S3 | через LB: 503; после восстановления: 200 | console |
|
||||||
## Эксперименты и отказы
|
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| TODO | TODO | TODO | TODO |
|
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
## Ограничения и следующие шаги
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
MinIO, PostgreSQL и Redis запущены по одному экземпляру. Cleanup предназначен для orphan-объектов; он не восстанавливает отсутствующие объекты для существующих метаданных. Административная консоль MinIO не публикуется в сеть.
|
||||||
|
|||||||
+175
@@ -0,0 +1,175 @@
|
|||||||
|
x-api: &api
|
||||||
|
image: mtusi2026lab4-api:local
|
||||||
|
build: .
|
||||||
|
dns_opt:
|
||||||
|
- timeout:1
|
||||||
|
- attempts:1
|
||||||
|
environment:
|
||||||
|
APP_HOST: 0.0.0.0
|
||||||
|
APP_PORT: 8080
|
||||||
|
POSTGRES_HOST: postgres
|
||||||
|
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||||
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||||
|
REDIS_URL: redis://redis:6379/0
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30}
|
||||||
|
MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536}
|
||||||
|
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
|
||||||
|
S3_ENDPOINT: http://s3:9000
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||||
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||||
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||||
|
networks: [backend]
|
||||||
|
cpus: 1.0
|
||||||
|
mem_limit: 512m
|
||||||
|
stop_grace_period: 10s
|
||||||
|
|
||||||
|
services:
|
||||||
|
api:
|
||||||
|
<<: *api
|
||||||
|
deploy:
|
||||||
|
replicas: 2
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
start_period: 5s
|
||||||
|
retries: 2
|
||||||
|
|
||||||
|
migrate:
|
||||||
|
<<: *api
|
||||||
|
command: [python, -m, src.database]
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
restart: 'no'
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
|
s3:
|
||||||
|
# RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest.
|
||||||
|
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||||
|
command: [server, /data, --console-address, '127.0.0.1:9001']
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||||
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||||
|
MINIO_BROWSER: 'off'
|
||||||
|
volumes:
|
||||||
|
- s3_data:/data
|
||||||
|
networks: [backend]
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
cpus: 1.0
|
||||||
|
mem_limit: 512m
|
||||||
|
|
||||||
|
s3-init:
|
||||||
|
# The pinned server image also contains mc; no second image is needed.
|
||||||
|
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||||
|
entrypoint: [sh, /bootstrap.sh]
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||||
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||||
|
volumes:
|
||||||
|
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
|
||||||
|
- ./infra/s3-policy.json:/policy.json:ro
|
||||||
|
depends_on:
|
||||||
|
s3:
|
||||||
|
condition: service_healthy
|
||||||
|
networks: [backend]
|
||||||
|
restart: 'no'
|
||||||
|
cpus: 0.25
|
||||||
|
mem_limit: 128m
|
||||||
|
|
||||||
|
cleanup:
|
||||||
|
<<: *api
|
||||||
|
command: [python, -m, src.cleanup]
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
restart: unless-stopped
|
||||||
|
cpus: 0.25
|
||||||
|
mem_limit: 128m
|
||||||
|
|
||||||
|
nginx:
|
||||||
|
image: nginx:1.28.0-alpine
|
||||||
|
entrypoint: [sh, /etc/nginx/readiness.sh]
|
||||||
|
dns_opt:
|
||||||
|
- timeout:1
|
||||||
|
- attempts:1
|
||||||
|
ports:
|
||||||
|
- '127.0.0.1:${APP_PORT:-8080}:8080'
|
||||||
|
volumes:
|
||||||
|
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
- ./infra/readiness.sh:/etc/nginx/readiness.sh:ro
|
||||||
|
networks: [edge, backend]
|
||||||
|
depends_on:
|
||||||
|
api:
|
||||||
|
condition: service_healthy
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 3
|
||||||
|
cpus: 0.5
|
||||||
|
mem_limit: 128m
|
||||||
|
stop_grace_period: 10s
|
||||||
|
|
||||||
|
postgres:
|
||||||
|
image: postgres:17.4-bookworm
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||||
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||||
|
volumes:
|
||||||
|
- postgres_data:/var/lib/postgresql/data
|
||||||
|
networks: [backend]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
cpus: 1.0
|
||||||
|
mem_limit: 512m
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:7.4.2-bookworm
|
||||||
|
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"]
|
||||||
|
volumes:
|
||||||
|
- redis_data:/data
|
||||||
|
networks: [backend]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
cpus: 0.5
|
||||||
|
mem_limit: 256m
|
||||||
|
|
||||||
|
networks:
|
||||||
|
edge:
|
||||||
|
backend:
|
||||||
|
internal: true
|
||||||
|
volumes:
|
||||||
|
postgres_data:
|
||||||
|
redis_data:
|
||||||
|
s3_data:
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# Перенесённая основа lab3
|
||||||
|
|
||||||
|
На этом этапе перенесена только работающая основа: FastAPI, PostgreSQL,
|
||||||
|
миграции 001–003, сессии и кеш Redis, идемпотентность сообщений, две API-реплики
|
||||||
|
и Nginx с проверкой готовности. S3 и вложения ещё не реализованы.
|
||||||
|
Контракт и публичные тесты остаются от lab4; полный smoke lab4 пока не пройдёт.
|
||||||
|
|
||||||
|
## Запуск основы
|
||||||
|
|
||||||
|
Скопируйте `.env.example` в `.env` и задайте свой `POSTGRES_PASSWORD`.
|
||||||
|
Параметры `*_FILE` и S3 пока являются настройками будущей реализации;
|
||||||
|
текущая основа использует `POSTGRES_*` и Redis из Compose.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose --env-file .env config --quiet
|
||||||
|
docker compose --env-file .env up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
API доступен через http://localhost:8080. При одновременно запущенной lab3
|
||||||
|
укажите другой `APP_PORT` в `.env`. Имя образа lab4 отдельное, volumes
|
||||||
|
изолируются именем Compose-проекта. Миграции выполняет сервис `migrate`.
|
||||||
|
|
||||||
|
Проверка перенесённой логики после запуска:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose exec -T api python -m unittest discover -s student-tests -v
|
||||||
|
```
|
||||||
|
|
||||||
|
В `student-tests` перенесены проверки базового API/сессий и идемпотентности.
|
||||||
|
Эксперименты масштабирования, старые результаты и отчёт lab3 не переносились.
|
||||||
|
Скрипты `scripts/restart.*` и `scripts/stop-one.*` работают с текущим проектом;
|
||||||
|
для остановки реплики требуется `TARGET_INSTANCE_ID` из `X-Instance-Id`.
|
||||||
|
При добавлении S3 потребуется включить его в сценарий перезапуска и адаптировать
|
||||||
|
лимиты тела запроса в API и Nginx для загрузки изображений.
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
worker_processes 1;
|
||||||
|
worker_shutdown_timeout 8s;
|
||||||
|
events { worker_connections 1024; }
|
||||||
|
http {
|
||||||
|
upstream messenger {
|
||||||
|
zone messenger 64k;
|
||||||
|
include /etc/nginx/ready-backends.conf;
|
||||||
|
}
|
||||||
|
log_format upstream '$request_method $uri $status upstream=$upstream_addr '
|
||||||
|
'upstream_status=$upstream_status time=$request_time';
|
||||||
|
access_log /var/log/nginx/access.log upstream;
|
||||||
|
server {
|
||||||
|
listen 8080;
|
||||||
|
# API enforces the byte limit and returns the contracted JSON error.
|
||||||
|
client_max_body_size 0;
|
||||||
|
client_body_timeout 5s;
|
||||||
|
client_header_timeout 5s;
|
||||||
|
keepalive_timeout 5s;
|
||||||
|
location / {
|
||||||
|
proxy_pass http://messenger;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header Connection close;
|
||||||
|
proxy_connect_timeout 1s;
|
||||||
|
proxy_send_timeout 35s;
|
||||||
|
proxy_read_timeout 120s;
|
||||||
|
# A 503 is an API response (for example, S3 is unavailable), so
|
||||||
|
# preserve it for the client instead of marking every replica down.
|
||||||
|
proxy_next_upstream error timeout http_502 http_504;
|
||||||
|
proxy_next_upstream_tries 2;
|
||||||
|
proxy_next_upstream_timeout 8s;
|
||||||
|
# Never transparently retry a POST already sent to an API.
|
||||||
|
# A client retries messages using the original Idempotency-Key.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Only ready API addresses enter the upstream, including at nginx startup.
|
||||||
|
update_backends() {
|
||||||
|
next=/etc/nginx/ready-backends.conf.next
|
||||||
|
: > "$next"
|
||||||
|
for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do
|
||||||
|
if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then
|
||||||
|
echo "server $address:8080;" >> "$next"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
# If every running API temporarily loses a shared dependency (for example
|
||||||
|
# S3), retain the last addresses. The API can then return the contracted
|
||||||
|
# 503 instead of Nginx replacing it with a 502. At initial startup there
|
||||||
|
# is no previous healthy set, so use an inert placeholder.
|
||||||
|
if [ ! -s "$next" ]; then
|
||||||
|
if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then
|
||||||
|
rm -f "$next"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
echo 'server 127.0.0.1:9 down;' > "$next"
|
||||||
|
fi
|
||||||
|
if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then
|
||||||
|
mv "$next" /etc/nginx/ready-backends.conf
|
||||||
|
if [ "${1:-}" = reload ] && [ -f /var/run/nginx.pid ]; then
|
||||||
|
nginx -s reload
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
rm -f "$next"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
update_backends
|
||||||
|
(
|
||||||
|
while sleep 2; do
|
||||||
|
update_backends reload
|
||||||
|
done
|
||||||
|
) &
|
||||||
|
exec nginx -g 'daemon off;'
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID"
|
||||||
|
mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||||
|
mc mb --ignore-existing local/messenger-attachments
|
||||||
|
mc anonymous set none local/messenger-attachments
|
||||||
|
mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null
|
||||||
|
mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json
|
||||||
|
mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID"
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||||
|
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
CREATE TABLE users (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||||
|
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||||
|
password_hash text NOT NULL,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chats (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chat_members (
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
PRIMARY KEY (chat_id, user_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||||
|
|
||||||
|
CREATE TABLE messages (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
sender_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||||
|
CREATE INDEX chats_order_idx ON chats (created_at, id);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0;
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
-- Additive migration: no rewrite of existing lab2 tables or messages.
|
||||||
|
CREATE TABLE message_idempotency (
|
||||||
|
actor_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||||
|
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
PRIMARY KEY (actor_id, chat_id, key)
|
||||||
|
);
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
CREATE TABLE attachments (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
owner_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
object_key text NOT NULL UNIQUE,
|
||||||
|
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||||
|
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||||
|
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||||
|
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
|
||||||
|
|
||||||
|
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
|
||||||
|
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||||
|
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
|
||||||
|
ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0);
|
||||||
|
ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10);
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
fastapi==0.115.6
|
||||||
|
uvicorn==0.34.0
|
||||||
|
pydantic==2.13.5
|
||||||
|
starlette==0.41.3
|
||||||
|
annotated-types==0.8.0
|
||||||
|
anyio==4.15.1
|
||||||
|
click==8.5.0
|
||||||
|
colorama==0.4.6; sys_platform == "win32"
|
||||||
|
h11==0.16.0
|
||||||
|
idna==3.20
|
||||||
|
pydantic_core==2.46.5
|
||||||
|
typing-inspection==0.4.4
|
||||||
|
typing_extensions==4.16.0
|
||||||
|
psycopg[binary]==3.2.6
|
||||||
|
SQLAlchemy==2.0.41
|
||||||
|
redis==5.2.1
|
||||||
|
argon2-cffi==23.1.0
|
||||||
|
boto3==1.40.70
|
||||||
|
Pillow==12.0.0
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""Project-scoped Docker operations; never read/export container environments."""
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def docker(*args):
|
||||||
|
return subprocess.check_output(['docker', *args], cwd=ROOT, text=True, timeout=120).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def compose(*args):
|
||||||
|
return docker('compose', *args)
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot(service=None):
|
||||||
|
ids = compose('ps', '-a', '-q', *([service] if service else [])).split()
|
||||||
|
if not ids:
|
||||||
|
return []
|
||||||
|
# Select fields before returning; no secrets in evidence.
|
||||||
|
template = ('{"id":{{json .Id}},"name":{{json .Name}},"instance":{{json .Config.Hostname}},'
|
||||||
|
'"image":{{json .Image}},"state":{{json .State.Status}},'
|
||||||
|
'"started":{{json .State.StartedAt}},'
|
||||||
|
'"health":{{with index .State "Health"}}{{json .Status}}{{else}}null{{end}},'
|
||||||
|
'"cpu":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},'
|
||||||
|
'"ports":{{json .HostConfig.PortBindings}}}')
|
||||||
|
return [json.loads(line) for line in docker('inspect', '--format', template, *ids).splitlines()]
|
||||||
|
|
||||||
|
|
||||||
|
def wait_api(count=2, timeout=60):
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
rows = snapshot('api')
|
||||||
|
if len(rows) == count and all(r['health'] == 'healthy' and r['state'] == 'running' for r in rows):
|
||||||
|
time.sleep(3) # Allow the nginx readiness poll (2s) and reload.
|
||||||
|
return rows
|
||||||
|
time.sleep(0.5)
|
||||||
|
raise RuntimeError('API replicas did not become healthy')
|
||||||
|
|
||||||
|
|
||||||
|
def save(name, value):
|
||||||
|
(ROOT / 'evidence').mkdir(exist_ok=True)
|
||||||
|
(ROOT / 'evidence' / name).write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8')
|
||||||
|
|
||||||
|
|
||||||
|
def stop_instance(instance):
|
||||||
|
before = snapshot('api')
|
||||||
|
matches = [r for r in before if r['instance'] == instance and r['state'] == 'running']
|
||||||
|
if len(matches) != 1 or sum(r['state'] == 'running' for r in before) < 2:
|
||||||
|
raise RuntimeError('Target must identify one running API with a running peer')
|
||||||
|
docker('stop', '--time', '8', matches[0]['id'])
|
||||||
|
result = {'target': instance, 'before': before, 'after': snapshot('api')}
|
||||||
|
if next(r for r in result['after'] if r['instance'] == instance)['state'] != 'exited':
|
||||||
|
raise RuntimeError('Target was not stopped')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def restore():
|
||||||
|
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
|
||||||
|
return wait_api()
|
||||||
|
|
||||||
|
|
||||||
|
def rolling():
|
||||||
|
before = wait_api()
|
||||||
|
if len(before) != 2:
|
||||||
|
raise RuntimeError('This rolling scenario requires exactly two replicas')
|
||||||
|
steps = []
|
||||||
|
for old in before:
|
||||||
|
stop_instance(old['instance'])
|
||||||
|
# Only remove the selected, already stopped API; never remove a volume.
|
||||||
|
docker('rm', old['id'])
|
||||||
|
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=2', 'api')
|
||||||
|
steps.append({'replaced': old['id'], 'ready': wait_api()})
|
||||||
|
result = {'before': before, 'steps': steps, 'after': snapshot('api')}
|
||||||
|
if {r['id'] for r in before} & {r['id'] for r in result['after']}:
|
||||||
|
raise RuntimeError('Not all original API containers were replaced')
|
||||||
|
return result
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
@echo off
|
||||||
|
cd /d "%~dp0.."
|
||||||
|
python scripts\restart.py
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
"""Restart the whole lab stack, retaining volumes and recording actual start times."""
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot():
|
||||||
|
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split()
|
||||||
|
return subprocess.check_output(
|
||||||
|
['docker', 'inspect', '--format', '{{.Name}} {{.State.StartedAt}}', *ids],
|
||||||
|
cwd=ROOT, text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
before = snapshot()
|
||||||
|
subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis', 's3', 'cleanup', 'nginx'],
|
||||||
|
cwd=ROOT, check=True, timeout=60)
|
||||||
|
after = snapshot()
|
||||||
|
(ROOT / 'evidence').mkdir(exist_ok=True)
|
||||||
|
(ROOT / 'evidence' / 'restart.txt').write_text(
|
||||||
|
'Command: docker compose restart api postgres redis s3 cleanup nginx\nBefore (UTC):\n' + before
|
||||||
|
+ 'After (UTC):\n' + after, encoding='utf-8')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
python3 scripts/restart.py
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
@echo off
|
||||||
|
cd /d "%~dp0.."
|
||||||
|
python scripts\stop-one.py
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
import os
|
||||||
|
from ops import stop_instance
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
stop_instance(os.environ['TARGET_INSTANCE_ID'])
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
exec python3 scripts/stop-one.py
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
"""Upload one fixture concurrently and record the peak RSS of API containers."""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT / 'tests'))
|
||||||
|
from client import Client, check
|
||||||
|
|
||||||
|
|
||||||
|
def api_memory_bytes():
|
||||||
|
text = subprocess.check_output(
|
||||||
|
['docker', 'compose', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', 'api'],
|
||||||
|
cwd=ROOT, text=True,
|
||||||
|
)
|
||||||
|
values = []
|
||||||
|
for line in text.splitlines():
|
||||||
|
amount = line.split()[1]
|
||||||
|
number, unit = float(amount[:-3]), amount[-3:].lower()
|
||||||
|
values.append(int(number * {'b': 1, 'kib': 1024, 'mib': 1024**2, 'gib': 1024**3}[unit]))
|
||||||
|
return sum(values)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||||
|
parser.add_argument('--concurrency', type=int, default=10)
|
||||||
|
parser.add_argument('--output', default='evidence/upload-load.json')
|
||||||
|
args = parser.parse_args()
|
||||||
|
check(args.concurrency >= 10, 'concurrency must be at least 10')
|
||||||
|
client = Client(args.base_url)
|
||||||
|
alice, _, _, chat = client.bootstrap()
|
||||||
|
content = (ROOT / 'tests' / 'fixtures' / 'sample.png').read_bytes()
|
||||||
|
start = threading.Event()
|
||||||
|
done = threading.Event()
|
||||||
|
peak = [0]
|
||||||
|
|
||||||
|
def monitor():
|
||||||
|
while not done.is_set():
|
||||||
|
peak[0] = max(peak[0], api_memory_bytes())
|
||||||
|
time.sleep(.2)
|
||||||
|
|
||||||
|
def upload(index):
|
||||||
|
start.wait()
|
||||||
|
return Client(args.base_url, timeout=60).call(
|
||||||
|
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=load-%d.png' % index,
|
||||||
|
201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||||
|
|
||||||
|
watcher = threading.Thread(target=monitor, daemon=True)
|
||||||
|
watcher.start()
|
||||||
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||||
|
jobs = [pool.submit(upload, item) for item in range(args.concurrency)]
|
||||||
|
start.set()
|
||||||
|
results = [job.result() for job in jobs]
|
||||||
|
done.set()
|
||||||
|
watcher.join(timeout=1)
|
||||||
|
report = {'concurrency': args.concurrency, 'fixture_bytes': len(content),
|
||||||
|
'successful_uploads': len(results), 'peak_api_memory_bytes': peak[0]}
|
||||||
|
destination = ROOT / args.output
|
||||||
|
destination.parent.mkdir(exist_ok=True)
|
||||||
|
destination.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
||||||
|
print(json.dumps(report, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import os
|
||||||
|
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
uvicorn.run(
|
||||||
|
'src.main:app',
|
||||||
|
host=os.getenv('APP_HOST', '0.0.0.0'),
|
||||||
|
port=int(os.getenv('APP_PORT', '8080')),
|
||||||
|
workers=1,
|
||||||
|
timeout_graceful_shutdown=5,
|
||||||
|
)
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
import os
|
||||||
|
import unicodedata
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from PIL import Image, UnidentifiedImageError
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from .errors import APIError
|
||||||
|
from .models import Attachment, ChatMember
|
||||||
|
from .object_storage import BUCKET, PREFIX, s3_client
|
||||||
|
from .store import ensure_uuid, serialize, store
|
||||||
|
|
||||||
|
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
|
||||||
|
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
|
||||||
|
if MAX_BYTES < 1 or MAX_PIXELS < 1:
|
||||||
|
raise ValueError('Upload limits must be positive')
|
||||||
|
|
||||||
|
|
||||||
|
def validate_filename(filename):
|
||||||
|
if (not filename or not filename.strip() or len(filename) > 255
|
||||||
|
or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)):
|
||||||
|
raise APIError(400, 'invalid_request', 'Неверное имя файла')
|
||||||
|
|
||||||
|
|
||||||
|
def validate_image(source, content_type):
|
||||||
|
try:
|
||||||
|
source.seek(0)
|
||||||
|
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
|
||||||
|
if picture.width * picture.height > MAX_PIXELS:
|
||||||
|
raise APIError(413, 'payload_too_large', 'Слишком много пикселей')
|
||||||
|
if Image.MIME[picture.format] != content_type:
|
||||||
|
raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type')
|
||||||
|
picture.verify()
|
||||||
|
source.seek(0)
|
||||||
|
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
|
||||||
|
picture.load() # verify alone does not decode JPEG or detect truncated pixel data.
|
||||||
|
except Image.DecompressionBombError as exc:
|
||||||
|
raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc
|
||||||
|
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
|
||||||
|
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc
|
||||||
|
source.seek(0)
|
||||||
|
|
||||||
|
|
||||||
|
def lock_object(session, key):
|
||||||
|
# Shared by upload and cleanup: no deletion while a COMMIT is in flight.
|
||||||
|
session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0))))
|
||||||
|
|
||||||
|
|
||||||
|
def metadata(attachment):
|
||||||
|
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
|
||||||
|
'content_type', 'size_bytes', 'sha256', 'created_at')
|
||||||
|
return {**result, 'updated_at': result['created_at'], 'status': 'ready',
|
||||||
|
'variants': [], 'error_code': None}
|
||||||
|
|
||||||
|
|
||||||
|
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
|
||||||
|
validate_image(source, content_type)
|
||||||
|
attachment_id = str(uuid4())
|
||||||
|
key = PREFIX + attachment_id
|
||||||
|
with store.sessions.begin() as session:
|
||||||
|
store._chat(session, chat_id, actor_id)
|
||||||
|
lock_object(session, key)
|
||||||
|
s3_client().put_object(Bucket=BUCKET, Key=key, Body=source,
|
||||||
|
ContentLength=size, ContentType=content_type)
|
||||||
|
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
|
||||||
|
object_key=key, filename=filename, content_type=content_type,
|
||||||
|
size_bytes=size, sha256=digest)
|
||||||
|
session.add(attachment)
|
||||||
|
session.flush()
|
||||||
|
result = metadata(attachment)
|
||||||
|
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def get_attachment(attachment_id, actor_id):
|
||||||
|
with store.sessions() as session:
|
||||||
|
attachment = session.scalar(select(Attachment).join(
|
||||||
|
ChatMember, ChatMember.chat_id == Attachment.chat_id).where(
|
||||||
|
Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id))
|
||||||
|
if attachment is None:
|
||||||
|
raise APIError(404, 'not_found', 'Объект не найден')
|
||||||
|
return attachment
|
||||||
+68
@@ -0,0 +1,68 @@
|
|||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from argon2 import PasswordHasher
|
||||||
|
from argon2.exceptions import VerificationError
|
||||||
|
from redis import Redis
|
||||||
|
|
||||||
|
from .errors import APIError
|
||||||
|
|
||||||
|
password_hasher = PasswordHasher()
|
||||||
|
dummy_hash = password_hasher.hash(secrets.token_urlsafe(32))
|
||||||
|
|
||||||
|
|
||||||
|
def verify_password(encoded, password):
|
||||||
|
try:
|
||||||
|
return password_hasher.verify(encoded or dummy_hash, password) and encoded is not None
|
||||||
|
except VerificationError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def unauthorized():
|
||||||
|
return APIError(401, 'unauthorized', 'Нет действительной идентификации')
|
||||||
|
|
||||||
|
|
||||||
|
class Sessions:
|
||||||
|
def __init__(self):
|
||||||
|
self.client = Redis.from_url(
|
||||||
|
os.getenv('REDIS_URL', 'redis://redis:6379/0'), decode_responses=True,
|
||||||
|
socket_connect_timeout=1, socket_timeout=1, max_connections=20,
|
||||||
|
retry_on_timeout=False,
|
||||||
|
)
|
||||||
|
self.ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
|
||||||
|
if self.ttl < 1:
|
||||||
|
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def key(token):
|
||||||
|
if not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||||
|
raise unauthorized()
|
||||||
|
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
|
||||||
|
|
||||||
|
def create(self, user):
|
||||||
|
token = secrets.token_urlsafe(32)
|
||||||
|
expires = datetime.now(timezone.utc) + timedelta(seconds=self.ttl)
|
||||||
|
expires_at = expires.isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||||
|
self.client.set(self.key(token), json.dumps({'user_id': user['id'], 'expires_at': expires_at}),
|
||||||
|
pxat=int(expires.timestamp() * 1000))
|
||||||
|
return {'token': token, 'token_type': 'Bearer', 'expires_at': expires_at, 'user': user}
|
||||||
|
|
||||||
|
def user_id(self, token):
|
||||||
|
raw = self.client.get(self.key(token))
|
||||||
|
if raw is None:
|
||||||
|
raise unauthorized()
|
||||||
|
session = json.loads(raw)
|
||||||
|
if datetime.fromisoformat(session['expires_at'].replace('Z', '+00:00')) <= datetime.now(timezone.utc):
|
||||||
|
raise unauthorized()
|
||||||
|
return session['user_id']
|
||||||
|
|
||||||
|
def delete(self, token):
|
||||||
|
if not self.client.delete(self.key(token)):
|
||||||
|
raise unauthorized()
|
||||||
|
|
||||||
|
|
||||||
|
sessions = Sessions()
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""History pages are disposable; PostgreSQL owns both data and their version."""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
from redis.exceptions import RedisError
|
||||||
|
|
||||||
|
from .auth import sessions
|
||||||
|
|
||||||
|
|
||||||
|
class HistoryCache:
|
||||||
|
def __init__(self, client, ttl=30, prefix='history'):
|
||||||
|
self.client = client
|
||||||
|
self.ttl = ttl
|
||||||
|
self.prefix = prefix
|
||||||
|
if ttl < 1:
|
||||||
|
raise ValueError('CACHE_TTL_SECONDS must be positive')
|
||||||
|
|
||||||
|
def key(self, chat_id, version, limit, after):
|
||||||
|
page = json.dumps([limit, after], separators=(',', ':'))
|
||||||
|
digest = hashlib.sha256(page.encode()).hexdigest()
|
||||||
|
return f'{self.prefix}:{chat_id}:{version}:{digest}'
|
||||||
|
|
||||||
|
def get_or_load(self, chat_id, version, limit, after, load):
|
||||||
|
key = self.key(chat_id, version, limit, after)
|
||||||
|
try:
|
||||||
|
cached = self.client.get(key)
|
||||||
|
except RedisError:
|
||||||
|
return load()
|
||||||
|
if cached is not None:
|
||||||
|
return json.loads(cached)
|
||||||
|
page = load()
|
||||||
|
try:
|
||||||
|
self.client.set(key, json.dumps(page, ensure_ascii=False), ex=self.ttl)
|
||||||
|
except RedisError:
|
||||||
|
pass
|
||||||
|
return page
|
||||||
|
|
||||||
|
|
||||||
|
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4')
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from .attachments import lock_object
|
||||||
|
from .database import create_db_engine
|
||||||
|
from .models import Attachment
|
||||||
|
from .object_storage import BUCKET, PREFIX, s3_client
|
||||||
|
|
||||||
|
GRACE = timedelta(hours=1)
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup(sessions, client, now=None):
|
||||||
|
cutoff = (now or datetime.now(timezone.utc)) - GRACE
|
||||||
|
deleted = 0
|
||||||
|
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
|
||||||
|
for obj in page.get('Contents', []):
|
||||||
|
if obj['LastModified'] >= cutoff:
|
||||||
|
continue
|
||||||
|
key = obj['Key']
|
||||||
|
with sessions.begin() as session:
|
||||||
|
lock_object(session, key)
|
||||||
|
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
|
||||||
|
continue
|
||||||
|
# Recheck age after taking the same lock as the uploader.
|
||||||
|
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
|
||||||
|
continue
|
||||||
|
client.delete_object(Bucket=BUCKET, Key=key)
|
||||||
|
deleted += 1
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('--once', action='store_true')
|
||||||
|
args = parser.parse_args()
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
engine = create_db_engine()
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
count = cleanup(sessionmaker(engine), s3_client())
|
||||||
|
logging.info('Cleanup completed: deleted=%d', count)
|
||||||
|
except Exception as exc:
|
||||||
|
logging.error('Cleanup deferred: %s', type(exc).__name__)
|
||||||
|
if args.once:
|
||||||
|
raise SystemExit(1) from None
|
||||||
|
if args.once:
|
||||||
|
break
|
||||||
|
time.sleep(3600)
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, func, insert, select
|
||||||
|
from sqlalchemy.engine import URL, make_url
|
||||||
|
|
||||||
|
from .models import SchemaMigration
|
||||||
|
|
||||||
|
|
||||||
|
def database_url():
|
||||||
|
if os.getenv('DATABASE_URL'):
|
||||||
|
url = make_url(os.environ['DATABASE_URL'])
|
||||||
|
if url.drivername in ('postgres', 'postgresql'):
|
||||||
|
url = url.set(drivername='postgresql+psycopg')
|
||||||
|
if url.drivername != 'postgresql+psycopg':
|
||||||
|
raise ValueError('DATABASE_URL must use PostgreSQL with psycopg')
|
||||||
|
return url
|
||||||
|
return URL.create(
|
||||||
|
'postgresql+psycopg', host=os.getenv('POSTGRES_HOST', 'postgres'),
|
||||||
|
database=os.getenv('POSTGRES_DB', 'messenger'),
|
||||||
|
username=os.getenv('POSTGRES_USER', 'messenger'),
|
||||||
|
password=os.environ['POSTGRES_PASSWORD'],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def create_db_engine():
|
||||||
|
return create_engine(
|
||||||
|
database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True,
|
||||||
|
connect_args={'connect_timeout': 2,
|
||||||
|
'options': '-c statement_timeout=3000 -c lock_timeout=2000'},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def migrate(engine):
|
||||||
|
with engine.begin() as conn:
|
||||||
|
conn.execute(select(func.pg_advisory_xact_lock(202602)))
|
||||||
|
SchemaMigration.__table__.create(conn, checkfirst=True)
|
||||||
|
applied = set(conn.scalars(select(SchemaMigration.version)))
|
||||||
|
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
|
||||||
|
if path.name not in applied:
|
||||||
|
conn.exec_driver_sql(path.read_text(encoding='utf-8'))
|
||||||
|
conn.execute(insert(SchemaMigration).values(version=path.name))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
engine = create_db_engine()
|
||||||
|
try:
|
||||||
|
migrate(engine)
|
||||||
|
print('Migrations applied')
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import Depends, Header, Query
|
||||||
|
|
||||||
|
from .auth import sessions, unauthorized
|
||||||
|
from .errors import APIError
|
||||||
|
from .store import store
|
||||||
|
|
||||||
|
|
||||||
|
def page_limit(limit: str = Query(default="50")) -> int:
|
||||||
|
if not limit.isascii() or not limit.isdecimal() or len(limit) > 3:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный limit")
|
||||||
|
number = int(limit)
|
||||||
|
if not 1 <= number <= 100:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный limit")
|
||||||
|
return number
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def bearer_token(authorization: Optional[str] = Header(default=None)) -> str:
|
||||||
|
parts = (authorization or '').split()
|
||||||
|
if len(parts) != 2 or parts[0].lower() != 'bearer':
|
||||||
|
raise unauthorized()
|
||||||
|
return parts[1]
|
||||||
|
|
||||||
|
|
||||||
|
def current_user(token: str = Depends(bearer_token)) -> dict:
|
||||||
|
user = store.get_user(sessions.user_id(token))
|
||||||
|
if user is None:
|
||||||
|
raise unauthorized()
|
||||||
|
return user
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
from fastapi import Request
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||||
|
from .middleware import INSTANCE_ID
|
||||||
|
|
||||||
|
|
||||||
|
class APIError(Exception):
|
||||||
|
def __init__(self, status_code: int, code: str, message: str):
|
||||||
|
self.status_code = status_code
|
||||||
|
self.code = code
|
||||||
|
self.message = message
|
||||||
|
|
||||||
|
|
||||||
|
def error_response(request: Request, status_code: int, code: str, message: str) -> JSONResponse:
|
||||||
|
request_id = getattr(request.state, "request_id", "")
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status_code,
|
||||||
|
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
|
||||||
|
content={
|
||||||
|
"error": {
|
||||||
|
"code": code,
|
||||||
|
"message": message,
|
||||||
|
"request_id": request_id,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def api_error_handler(request: Request, exc: APIError) -> JSONResponse:
|
||||||
|
response = error_response(request, exc.status_code, exc.code, exc.message)
|
||||||
|
if exc.status_code == 429:
|
||||||
|
response.headers['Retry-After'] = '1'
|
||||||
|
if exc.status_code == 401:
|
||||||
|
response.headers['WWW-Authenticate'] = (
|
||||||
|
'Basic realm="messenger", charset="UTF-8"'
|
||||||
|
if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def dependency_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||||
|
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
|
||||||
|
|
||||||
|
|
||||||
|
def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||||
|
return error_response(request, 400, "invalid_request", "Неверный запрос")
|
||||||
|
|
||||||
|
|
||||||
|
def http_error_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
|
||||||
|
if exc.status_code == 404:
|
||||||
|
return error_response(request, 404, "not_found", "Объект не найден")
|
||||||
|
if exc.status_code == 405:
|
||||||
|
return error_response(request, 400, "invalid_request", "Метод не поддерживается")
|
||||||
|
else:
|
||||||
|
return error_response(request, exc.status_code, "invalid_request", "Неверный запрос")
|
||||||
|
|
||||||
|
|
||||||
|
def internal_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||||
|
return error_response(request, 500, "internal_error", "Внутренняя ошибка")
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from starlette.requests import Request
|
||||||
|
|
||||||
|
from .errors import error_response
|
||||||
|
from .middleware import is_upload
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Limits:
|
||||||
|
max_users: int = 1000
|
||||||
|
max_chats: int = 1000
|
||||||
|
max_messages: int = 10000
|
||||||
|
max_text_bytes: int = 16 * 1024 * 1024
|
||||||
|
max_request_bytes: int = 64 * 1024
|
||||||
|
max_inflight: int = 32
|
||||||
|
message_rate: int = 1000
|
||||||
|
message_burst: int = 1000
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls):
|
||||||
|
values = {}
|
||||||
|
for name, default in cls().__dict__.items():
|
||||||
|
value = int(os.getenv(name.upper(), str(default)))
|
||||||
|
if value < 1:
|
||||||
|
raise ValueError(f'{name.upper()} must be positive')
|
||||||
|
values[name] = value
|
||||||
|
return cls(**values)
|
||||||
|
|
||||||
|
|
||||||
|
limits = Limits.from_env()
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceLimitMiddleware:
|
||||||
|
def __init__(self, app, settings=limits):
|
||||||
|
self.app = app
|
||||||
|
self.settings = settings
|
||||||
|
self.active = 0
|
||||||
|
|
||||||
|
async def __call__(self, scope, receive, send):
|
||||||
|
if scope['type'] != 'http' or scope['path'].startswith('/health/'):
|
||||||
|
return await self.app(scope, receive, send)
|
||||||
|
|
||||||
|
request = Request(scope)
|
||||||
|
|
||||||
|
async def reject(status, code, message):
|
||||||
|
response = error_response(request, status, code, message)
|
||||||
|
if status == 503:
|
||||||
|
response.headers['Retry-After'] = '1'
|
||||||
|
await response(scope, receive, send)
|
||||||
|
|
||||||
|
if self.active >= self.settings.max_inflight:
|
||||||
|
return await reject(503, 'unavailable', 'Слишком много одновременных запросов')
|
||||||
|
self.active += 1
|
||||||
|
try:
|
||||||
|
# The upload endpoint authenticates and bounds its stream itself.
|
||||||
|
if is_upload(scope['method'], scope['path']):
|
||||||
|
return await self.app(scope, receive, send)
|
||||||
|
length = request.headers.get('content-length')
|
||||||
|
if length is not None:
|
||||||
|
try:
|
||||||
|
size = int(length)
|
||||||
|
if size < 0:
|
||||||
|
raise ValueError()
|
||||||
|
except ValueError:
|
||||||
|
return await reject(400, 'invalid_request', 'Неверный Content-Length')
|
||||||
|
if size > self.settings.max_request_bytes:
|
||||||
|
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
|
||||||
|
|
||||||
|
body = bytearray()
|
||||||
|
try:
|
||||||
|
async with asyncio.timeout(5):
|
||||||
|
while True:
|
||||||
|
message = await receive()
|
||||||
|
if message['type'] == 'http.disconnect':
|
||||||
|
return
|
||||||
|
chunk = message.get('body', b'')
|
||||||
|
if len(body) + len(chunk) > self.settings.max_request_bytes:
|
||||||
|
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
|
||||||
|
body.extend(chunk)
|
||||||
|
if not message.get('more_body', False):
|
||||||
|
break
|
||||||
|
except TimeoutError:
|
||||||
|
return await reject(503, 'unavailable', 'Истекло время чтения запроса')
|
||||||
|
|
||||||
|
delivered = False
|
||||||
|
|
||||||
|
async def buffered_receive():
|
||||||
|
nonlocal delivered
|
||||||
|
if not delivered:
|
||||||
|
delivered = True
|
||||||
|
return {'type': 'http.request', 'body': bytes(body), 'more_body': False}
|
||||||
|
return await receive()
|
||||||
|
|
||||||
|
await self.app(scope, buffered_receive, send)
|
||||||
|
finally:
|
||||||
|
self.active -= 1
|
||||||
+48
@@ -0,0 +1,48 @@
|
|||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from starlette.exceptions import HTTPException
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
from redis.exceptions import RedisError
|
||||||
|
from botocore.exceptions import BotoCoreError, ClientError
|
||||||
|
|
||||||
|
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler
|
||||||
|
from .database import create_db_engine
|
||||||
|
from .store import store
|
||||||
|
from .auth import sessions
|
||||||
|
from .middleware import RequestIdMiddleware
|
||||||
|
from .limits import ResourceLimitMiddleware
|
||||||
|
from .routers.health import router as health_router
|
||||||
|
from .routers.users import router as users_router
|
||||||
|
from .routers.chats import router as chats_router
|
||||||
|
from .routers.auth import router as auth_router
|
||||||
|
from .routers.attachments import router as attachments_router
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(app):
|
||||||
|
engine = create_db_engine()
|
||||||
|
try:
|
||||||
|
store.sessions = sessionmaker(engine, expire_on_commit=False)
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
sessions.client.close()
|
||||||
|
|
||||||
|
|
||||||
|
app = FastAPI(title="MTUSI Messenger", lifespan=lifespan)
|
||||||
|
app.add_middleware(ResourceLimitMiddleware)
|
||||||
|
app.add_middleware(RequestIdMiddleware)
|
||||||
|
app.add_exception_handler(APIError, api_error_handler)
|
||||||
|
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
||||||
|
app.add_exception_handler(HTTPException, http_error_handler)
|
||||||
|
app.add_exception_handler(Exception, internal_error_handler)
|
||||||
|
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError):
|
||||||
|
app.add_exception_handler(error, dependency_error_handler)
|
||||||
|
|
||||||
|
app.include_router(health_router)
|
||||||
|
app.include_router(users_router)
|
||||||
|
app.include_router(chats_router)
|
||||||
|
app.include_router(auth_router)
|
||||||
|
app.include_router(attachments_router)
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
from uuid import uuid4
|
||||||
|
from socket import gethostname
|
||||||
|
from starlette.responses import JSONResponse
|
||||||
|
from starlette.middleware.base import BaseHTTPMiddleware
|
||||||
|
from starlette.requests import Request
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
def is_upload(method, path):
|
||||||
|
return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None
|
||||||
|
|
||||||
|
INSTANCE_ID = gethostname()
|
||||||
|
|
||||||
|
|
||||||
|
class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||||
|
async def dispatch(self, request: Request, call_next):
|
||||||
|
request_id = request.headers.get("X-Request-Id", "").strip() or str(uuid4())
|
||||||
|
request.state.request_id = request_id
|
||||||
|
|
||||||
|
if (request.method in {"POST", "PUT", "PATCH"}
|
||||||
|
and request.url.path.startswith("/api/")
|
||||||
|
and not is_upload(request.method, request.url.path)
|
||||||
|
and request.url.path != '/api/v1/auth/sessions'):
|
||||||
|
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||||
|
if content_type != "application/json":
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=415,
|
||||||
|
content={
|
||||||
|
"error": {
|
||||||
|
"code": "unsupported_media_type",
|
||||||
|
"message": "Ожидается Content-Type application/json",
|
||||||
|
"request_id": request_id,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
response.headers["X-Request-Id"] = request_id
|
||||||
|
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||||
|
return response
|
||||||
|
|
||||||
|
response = await call_next(request)
|
||||||
|
response.headers["X-Request-Id"] = request_id
|
||||||
|
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||||
|
return response
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
|
||||||
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||||
|
from sqlalchemy.dialects.postgresql import ARRAY
|
||||||
|
|
||||||
|
|
||||||
|
class Base(DeclarativeBase):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class User(Base):
|
||||||
|
__tablename__ = 'users'
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"),
|
||||||
|
CheckConstraint('char_length(display_name) BETWEEN 1 AND 100'),
|
||||||
|
)
|
||||||
|
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||||
|
username: Mapped[str] = mapped_column(Text, unique=True)
|
||||||
|
display_name: Mapped[str] = mapped_column(Text)
|
||||||
|
password_hash: Mapped[str] = mapped_column(Text)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
class Chat(Base):
|
||||||
|
__tablename__ = 'chats'
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint('char_length(title) BETWEEN 1 AND 100'),
|
||||||
|
Index('chats_order_idx', 'created_at', 'id'),
|
||||||
|
)
|
||||||
|
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||||
|
title: Mapped[str] = mapped_column(Text)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
history_version: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||||
|
|
||||||
|
|
||||||
|
class ChatMember(Base):
|
||||||
|
__tablename__ = 'chat_members'
|
||||||
|
__table_args__ = (Index('chat_members_user_idx', 'user_id', 'chat_id'),)
|
||||||
|
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||||
|
user_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
|
||||||
|
|
||||||
|
|
||||||
|
class Message(Base):
|
||||||
|
__tablename__ = 'messages'
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint('char_length(text) <= 4000'),
|
||||||
|
CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"),
|
||||||
|
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
|
||||||
|
)
|
||||||
|
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||||
|
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
|
||||||
|
sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
|
||||||
|
text: Mapped[str] = mapped_column(Text)
|
||||||
|
attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}')
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
class SchemaMigration(Base):
|
||||||
|
__tablename__ = 'schema_migrations'
|
||||||
|
version: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||||
|
applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class Attachment(Base):
|
||||||
|
__tablename__ = 'attachments'
|
||||||
|
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True)
|
||||||
|
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
|
||||||
|
owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
|
||||||
|
object_key: Mapped[str] = mapped_column(Text, unique=True)
|
||||||
|
filename: Mapped[str] = mapped_column(Text)
|
||||||
|
content_type: Mapped[str] = mapped_column(Text)
|
||||||
|
size_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||||
|
sha256: Mapped[str] = mapped_column(Text)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
class IdempotencyRecord(Base):
|
||||||
|
__tablename__ = 'message_idempotency'
|
||||||
|
actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
|
||||||
|
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||||
|
key: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||||
|
message_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('messages.id', ondelete='CASCADE'))
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
import os
|
||||||
|
from functools import lru_cache
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||||
|
PREFIX = 'originals/'
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def s3_client():
|
||||||
|
return boto3.client(
|
||||||
|
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||||
|
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
||||||
|
aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'],
|
||||||
|
aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'],
|
||||||
|
config=Config(connect_timeout=2, read_timeout=5,
|
||||||
|
retries={'total_max_attempts': 1},
|
||||||
|
s3={'addressing_style': 'path'},
|
||||||
|
request_checksum_calculation='when_required',
|
||||||
|
response_checksum_validation='when_required'),
|
||||||
|
)
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
from tempfile import TemporaryFile
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Query, Request
|
||||||
|
from starlette.background import BackgroundTask
|
||||||
|
from starlette.concurrency import run_in_threadpool
|
||||||
|
from starlette.requests import ClientDisconnect
|
||||||
|
from starlette.responses import StreamingResponse
|
||||||
|
|
||||||
|
from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename
|
||||||
|
from ..dependencies import current_user
|
||||||
|
from ..errors import APIError
|
||||||
|
from ..object_storage import BUCKET, s3_client
|
||||||
|
from ..store import ensure_uuid, store
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/api/v1', tags=['attachments'])
|
||||||
|
# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk.
|
||||||
|
decode_slot = asyncio.Semaphore(1)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('/chats/{chat_id}/attachments', status_code=201)
|
||||||
|
async def upload(chat_id: str, request: Request, filename: str = Query(...),
|
||||||
|
user: dict = Depends(current_user)):
|
||||||
|
chat_id = ensure_uuid(chat_id)
|
||||||
|
await run_in_threadpool(store.get_chat_for_user, chat_id, user['id'])
|
||||||
|
validate_filename(filename)
|
||||||
|
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
|
||||||
|
if content_type not in ('image/png', 'image/jpeg'):
|
||||||
|
raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG')
|
||||||
|
length = request.headers.get('content-length')
|
||||||
|
if length is not None:
|
||||||
|
try:
|
||||||
|
length = int(length)
|
||||||
|
if length < 0:
|
||||||
|
raise ValueError()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc
|
||||||
|
if length > MAX_BYTES:
|
||||||
|
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
|
||||||
|
size = 0
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with TemporaryFile() as source:
|
||||||
|
try:
|
||||||
|
async with asyncio.timeout(30):
|
||||||
|
async for chunk in request.stream():
|
||||||
|
size += len(chunk)
|
||||||
|
if size > MAX_BYTES:
|
||||||
|
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
|
||||||
|
digest.update(chunk)
|
||||||
|
await run_in_threadpool(source.write, chunk)
|
||||||
|
except (TimeoutError, ClientDisconnect) as exc:
|
||||||
|
raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc
|
||||||
|
if length is not None and size != length:
|
||||||
|
raise APIError(400, 'invalid_request', 'Неполное тело запроса')
|
||||||
|
async with decode_slot:
|
||||||
|
return await run_in_threadpool(save_attachment, source, chat_id, user['id'],
|
||||||
|
filename, content_type, size, digest.hexdigest())
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/attachments/{attachment_id}')
|
||||||
|
def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
|
||||||
|
return metadata(get_attachment(attachment_id, user['id']))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/attachments/{attachment_id}/content')
|
||||||
|
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
|
||||||
|
attachment = get_attachment(attachment_id, user['id'])
|
||||||
|
if variant != 'original':
|
||||||
|
raise APIError(404, 'not_found', 'Вариант не найден')
|
||||||
|
body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body']
|
||||||
|
|
||||||
|
def chunks():
|
||||||
|
try:
|
||||||
|
yield from body.iter_chunks(chunk_size=64 * 1024)
|
||||||
|
finally:
|
||||||
|
body.close()
|
||||||
|
|
||||||
|
return StreamingResponse(chunks(), media_type=attachment.content_type,
|
||||||
|
headers={'Content-Length': str(attachment.size_bytes),
|
||||||
|
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
|
||||||
|
'X-Content-Type-Options': 'nosniff'},
|
||||||
|
background=BackgroundTask(body.close))
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import base64
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Header, Request, Response
|
||||||
|
from starlette.concurrency import run_in_threadpool
|
||||||
|
|
||||||
|
from ..auth import sessions, unauthorized
|
||||||
|
from ..dependencies import bearer_token, current_user
|
||||||
|
from ..errors import APIError
|
||||||
|
from ..store import store
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('', status_code=201)
|
||||||
|
async def create_session(request: Request, authorization: str = Header(default='')):
|
||||||
|
if await request.body():
|
||||||
|
raise APIError(400, 'invalid_request', 'Вход не принимает тело запроса')
|
||||||
|
try:
|
||||||
|
scheme, encoded = authorization.split()
|
||||||
|
if scheme.lower() != 'basic':
|
||||||
|
raise ValueError()
|
||||||
|
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||||
|
if not username.isascii() or not 12 <= len(password) <= 128 or '\x00' in username:
|
||||||
|
raise ValueError()
|
||||||
|
except (ValueError, UnicodeError):
|
||||||
|
raise unauthorized() from None
|
||||||
|
user = await run_in_threadpool(store.login, username, password)
|
||||||
|
return await run_in_threadpool(sessions.create, user)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete('/current', status_code=204)
|
||||||
|
def logout(token: str = Depends(bearer_token), user: dict = Depends(current_user)):
|
||||||
|
sessions.delete(token)
|
||||||
|
return Response(status_code=204)
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
from typing import Optional
|
||||||
|
import re
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Header, Query, status
|
||||||
|
from ..errors import APIError
|
||||||
|
|
||||||
|
from ..dependencies import current_user, page_limit
|
||||||
|
from ..schemas.chat import Chat, ChatPage, CreateChat
|
||||||
|
from ..schemas.message import CreateMessage, Message, MessagePage
|
||||||
|
from ..store import store
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", response_model=Chat, status_code=status.HTTP_201_CREATED)
|
||||||
|
def create_chat(payload: CreateChat, user: dict = Depends(current_user)):
|
||||||
|
return store.create_chat(user["id"], payload.title, payload.member_ids)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=ChatPage)
|
||||||
|
def list_chats(
|
||||||
|
limit: int = Depends(page_limit),
|
||||||
|
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||||
|
user: dict = Depends(current_user),
|
||||||
|
):
|
||||||
|
return store.list_chats(user["id"], limit, cursor)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{chat_id}", response_model=Chat)
|
||||||
|
def get_chat(chat_id: str, user: dict = Depends(current_user)):
|
||||||
|
return store.get_chat_for_user(chat_id, user["id"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{chat_id}/messages", response_model=Message, status_code=status.HTTP_201_CREATED)
|
||||||
|
def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(current_user),
|
||||||
|
idempotency_key: Optional[str] = Header(default=None)):
|
||||||
|
if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key):
|
||||||
|
raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key')
|
||||||
|
return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{chat_id}/messages", response_model=MessagePage)
|
||||||
|
def list_messages(
|
||||||
|
chat_id: str,
|
||||||
|
limit: int = Depends(page_limit),
|
||||||
|
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||||
|
user: dict = Depends(current_user),
|
||||||
|
):
|
||||||
|
return store.list_messages(chat_id, user["id"], limit, cursor)
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
from fastapi import APIRouter
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from ..schemas.common import Health
|
||||||
|
from ..auth import sessions
|
||||||
|
from ..store import store
|
||||||
|
from ..object_storage import BUCKET, s3_client
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/health')
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/live", response_model=Health)
|
||||||
|
def health_live():
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/ready", response_model=Health)
|
||||||
|
def health_ready():
|
||||||
|
with store.sessions() as session:
|
||||||
|
session.execute(select(1))
|
||||||
|
sessions.client.ping()
|
||||||
|
s3_client().head_bucket(Bucket=BUCKET)
|
||||||
|
return {"status": "ok"}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
from fastapi import APIRouter, Depends, status
|
||||||
|
|
||||||
|
from ..dependencies import current_user
|
||||||
|
from ..schemas.user import CreateUser, User
|
||||||
|
from ..store import store
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", response_model=User, status_code=status.HTTP_201_CREATED)
|
||||||
|
def create_user(payload: CreateUser):
|
||||||
|
return store.create_user(payload.username, payload.display_name, payload.password)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/me", response_model=User)
|
||||||
|
def get_current_user(user: dict = Depends(current_user)):
|
||||||
|
return user
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from pydantic import Field, StrictStr, field_validator
|
||||||
|
|
||||||
|
from .common import ContractModel
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class CreateChat(ContractModel):
|
||||||
|
title: StrictStr = Field(min_length=1, max_length=100)
|
||||||
|
member_ids: List[StrictStr] = Field(min_length=1, max_length=100)
|
||||||
|
|
||||||
|
@field_validator("title")
|
||||||
|
def validate_title(cls, value):
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError("blank title")
|
||||||
|
return value
|
||||||
|
|
||||||
|
@field_validator("member_ids")
|
||||||
|
def validate_member_ids(cls, value):
|
||||||
|
if len(value) != len(set(value)):
|
||||||
|
raise ValueError("duplicate member_ids")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class Chat(ContractModel):
|
||||||
|
id: str
|
||||||
|
title: str
|
||||||
|
member_ids: List[str]
|
||||||
|
created_at: str
|
||||||
|
|
||||||
|
|
||||||
|
class ChatPage(ContractModel):
|
||||||
|
items: List[Chat]
|
||||||
|
next_cursor: Optional[str]
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from pydantic import BaseModel, ConfigDict, field_validator
|
||||||
|
|
||||||
|
|
||||||
|
class ContractModel(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
@field_validator('*')
|
||||||
|
def validate_text_encoding(cls, value, info):
|
||||||
|
if isinstance(value, str):
|
||||||
|
value.encode('utf-8')
|
||||||
|
if info.field_name != 'password' and '\x00' in value:
|
||||||
|
raise ValueError('NUL is not supported in PostgreSQL text')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class Health(ContractModel):
|
||||||
|
status: str
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from pydantic import Field, StrictStr, field_validator, model_validator
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from .common import ContractModel
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class CreateMessage(ContractModel):
|
||||||
|
text: StrictStr = Field(max_length=4000)
|
||||||
|
attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10)
|
||||||
|
|
||||||
|
@field_validator('attachment_ids')
|
||||||
|
def validate_ids(cls, value):
|
||||||
|
ids = [str(UUID(item)) for item in value]
|
||||||
|
if len(ids) != len(set(ids)):
|
||||||
|
raise ValueError('duplicate attachments')
|
||||||
|
return ids
|
||||||
|
|
||||||
|
@model_validator(mode='after')
|
||||||
|
def validate_content(self):
|
||||||
|
if not self.text.strip() and not self.attachment_ids:
|
||||||
|
raise ValueError('empty message')
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class Message(ContractModel):
|
||||||
|
id: str
|
||||||
|
chat_id: str
|
||||||
|
sender_id: str
|
||||||
|
text: str
|
||||||
|
attachment_ids: List[str]
|
||||||
|
created_at: str
|
||||||
|
|
||||||
|
|
||||||
|
class MessagePage(ContractModel):
|
||||||
|
items: List[Message]
|
||||||
|
next_cursor: Optional[str]
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
|
from pydantic import Field, StrictStr, field_validator
|
||||||
|
|
||||||
|
from .common import ContractModel
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||||
|
|
||||||
|
|
||||||
|
class CreateUser(ContractModel):
|
||||||
|
username: StrictStr
|
||||||
|
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||||
|
password: StrictStr = Field(min_length=12, max_length=128)
|
||||||
|
|
||||||
|
@field_validator("username")
|
||||||
|
def validate_username(cls, value):
|
||||||
|
if not USERNAME_RE.fullmatch(value):
|
||||||
|
raise ValueError("invalid username")
|
||||||
|
return value
|
||||||
|
|
||||||
|
@field_validator("display_name")
|
||||||
|
def validate_display_name(cls, value):
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError("blank display_name")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class User(ContractModel):
|
||||||
|
id: str
|
||||||
|
username: str
|
||||||
|
display_name: str = Field(min_length=1, max_length=100)
|
||||||
|
created_at: str
|
||||||
+226
@@ -0,0 +1,226 @@
|
|||||||
|
import base64
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Tuple
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import select, tuple_, update
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
|
||||||
|
from .auth import password_hasher, verify_password, unauthorized
|
||||||
|
from .errors import APIError
|
||||||
|
from .cache import history_cache
|
||||||
|
from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_uuid(value):
|
||||||
|
try:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ValueError()
|
||||||
|
return str(UUID(value))
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise APIError(400, 'invalid_request', 'Неверный UUID') from exc
|
||||||
|
|
||||||
|
|
||||||
|
def serialize(model, *fields):
|
||||||
|
if model is None:
|
||||||
|
return None
|
||||||
|
result = {}
|
||||||
|
for field in fields:
|
||||||
|
value = getattr(model, field)
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
value = value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||||
|
result[field] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
USER_FIELDS = ('id', 'username', 'display_name', 'created_at')
|
||||||
|
CHAT_FIELDS = ('id', 'title', 'created_at')
|
||||||
|
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids')
|
||||||
|
|
||||||
|
|
||||||
|
def after_condition(model, after):
|
||||||
|
return tuple_(model.created_at, model.id) > (
|
||||||
|
datetime.fromisoformat(after[0].replace('Z', '+00:00')), after[1])
|
||||||
|
|
||||||
|
|
||||||
|
def history_query(chat_id, limit, after):
|
||||||
|
query = select(Message).where(Message.chat_id == chat_id)
|
||||||
|
if after:
|
||||||
|
query = query.where(after_condition(Message, after))
|
||||||
|
return query.order_by(Message.created_at, Message.id).limit(limit + 1)
|
||||||
|
|
||||||
|
|
||||||
|
class PostgresStore:
|
||||||
|
def __init__(self, cache=history_cache):
|
||||||
|
self.sessions = None
|
||||||
|
self.cache = cache
|
||||||
|
|
||||||
|
def create_user(self, username, display_name, password):
|
||||||
|
user = User(username=username, display_name=display_name, password_hash=password_hasher.hash(password))
|
||||||
|
try:
|
||||||
|
with self.sessions.begin() as session:
|
||||||
|
session.add(user)
|
||||||
|
session.flush()
|
||||||
|
result = serialize(user, *USER_FIELDS)
|
||||||
|
return result
|
||||||
|
except IntegrityError as exc:
|
||||||
|
if getattr(exc.orig, 'sqlstate', None) == '23505':
|
||||||
|
raise APIError(409, 'conflict', 'Имя пользователя уже занято') from exc
|
||||||
|
raise
|
||||||
|
|
||||||
|
def get_user(self, user_id):
|
||||||
|
with self.sessions() as session:
|
||||||
|
return serialize(session.get(User, ensure_uuid(user_id)), *USER_FIELDS)
|
||||||
|
|
||||||
|
def login(self, username, password):
|
||||||
|
with self.sessions() as session:
|
||||||
|
user = session.scalar(select(User).where(User.username == username))
|
||||||
|
encoded = user.password_hash if user else None
|
||||||
|
result = serialize(user, *USER_FIELDS)
|
||||||
|
if not verify_password(encoded, password):
|
||||||
|
raise unauthorized()
|
||||||
|
return result
|
||||||
|
|
||||||
|
def create_chat(self, actor_id, title, member_ids):
|
||||||
|
members = [ensure_uuid(member) for member in member_ids]
|
||||||
|
if len(members) != len(set(members)) or actor_id not in members:
|
||||||
|
raise APIError(400, 'invalid_request', 'Неверный список участников')
|
||||||
|
try:
|
||||||
|
with self.sessions.begin() as session:
|
||||||
|
chat = Chat(title=title)
|
||||||
|
session.add(chat)
|
||||||
|
session.flush()
|
||||||
|
session.add_all([ChatMember(chat_id=chat.id, user_id=member) for member in members])
|
||||||
|
result = serialize(chat, *CHAT_FIELDS)
|
||||||
|
result['member_ids'] = sorted(members)
|
||||||
|
return result
|
||||||
|
except IntegrityError as exc:
|
||||||
|
if getattr(exc.orig, 'sqlstate', None) == '23503':
|
||||||
|
raise APIError(400, 'invalid_request', 'Неизвестный участник') from exc
|
||||||
|
raise
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _chat(session, chat_id, actor_id):
|
||||||
|
chat = session.scalar(select(Chat).join(ChatMember).where(
|
||||||
|
Chat.id == chat_id, ChatMember.user_id == actor_id))
|
||||||
|
if chat is None:
|
||||||
|
raise APIError(404, 'not_found', 'Объект не найден')
|
||||||
|
return chat
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _members(session, chat):
|
||||||
|
result = serialize(chat, *CHAT_FIELDS)
|
||||||
|
result['member_ids'] = list(session.scalars(select(ChatMember.user_id).where(
|
||||||
|
ChatMember.chat_id == chat.id).order_by(ChatMember.user_id)))
|
||||||
|
return result
|
||||||
|
|
||||||
|
def get_chat_for_user(self, chat_id, actor_id):
|
||||||
|
with self.sessions() as session:
|
||||||
|
return self._members(session, self._chat(session, ensure_uuid(chat_id), actor_id))
|
||||||
|
|
||||||
|
def list_chats(self, actor_id, limit, cursor):
|
||||||
|
after = self._decode_cursor(cursor, 'chats', actor_id) if cursor else None
|
||||||
|
query = select(Chat).join(ChatMember).where(ChatMember.user_id == actor_id)
|
||||||
|
if after:
|
||||||
|
query = query.where(after_condition(Chat, after))
|
||||||
|
query = query.order_by(Chat.created_at, Chat.id).limit(limit + 1)
|
||||||
|
with self.sessions() as session:
|
||||||
|
chats = list(session.scalars(query))
|
||||||
|
items = [self._members(session, chat) for chat in chats[:limit]]
|
||||||
|
return self._page(items, len(chats) > limit, 'chats', actor_id)
|
||||||
|
|
||||||
|
def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None):
|
||||||
|
chat_id = ensure_uuid(chat_id)
|
||||||
|
attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])]
|
||||||
|
with self.sessions.begin() as session:
|
||||||
|
self._chat(session, chat_id, actor_id)
|
||||||
|
if attachment_ids:
|
||||||
|
found = list(session.scalars(select(Attachment.id).where(
|
||||||
|
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)))
|
||||||
|
if len(found) != len(attachment_ids):
|
||||||
|
raise APIError(404, 'not_found', 'Объект не найден')
|
||||||
|
if idempotency_key is not None:
|
||||||
|
# Serialize writers to this chat before looking up the key.
|
||||||
|
# The same row is updated below for history cache invalidation.
|
||||||
|
session.execute(select(Chat.id).where(Chat.id == chat_id).with_for_update())
|
||||||
|
record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key))
|
||||||
|
if record is not None:
|
||||||
|
previous = session.get(Message, record.message_id)
|
||||||
|
if previous.text != text or previous.attachment_ids != attachment_ids:
|
||||||
|
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
|
||||||
|
return serialize(previous, *MESSAGE_FIELDS)
|
||||||
|
session.execute(update(Chat).where(Chat.id == chat_id).values(
|
||||||
|
history_version=Chat.history_version + 1))
|
||||||
|
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
|
||||||
|
session.add(message)
|
||||||
|
session.flush()
|
||||||
|
if idempotency_key is not None:
|
||||||
|
session.add(IdempotencyRecord(actor_id=actor_id, chat_id=chat_id,
|
||||||
|
key=idempotency_key, message_id=message.id))
|
||||||
|
result = serialize(message, *MESSAGE_FIELDS)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def list_messages(self, chat_id, actor_id, limit, cursor):
|
||||||
|
chat_id = ensure_uuid(chat_id)
|
||||||
|
after = self._decode_cursor(cursor, 'messages', chat_id) if cursor else None
|
||||||
|
with self.sessions() as session:
|
||||||
|
version = self._chat(session, chat_id, actor_id).history_version
|
||||||
|
return self.cache.get_or_load(
|
||||||
|
chat_id, version, limit, after,
|
||||||
|
lambda: self._load_messages(chat_id, limit, after),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _load_messages(self, chat_id, limit, after):
|
||||||
|
with self.sessions() as session:
|
||||||
|
messages = list(session.scalars(history_query(chat_id, limit, after)))
|
||||||
|
items = [serialize(message, *MESSAGE_FIELDS) for message in messages[:limit]]
|
||||||
|
return self._page(items, len(messages) > limit, 'messages', chat_id)
|
||||||
|
|
||||||
|
def _page(self, items, has_more, resource, owner_id):
|
||||||
|
cursor = None
|
||||||
|
if has_more:
|
||||||
|
last = items[-1]
|
||||||
|
cursor = self._encode_cursor(resource, owner_id, last['created_at'], last['id'])
|
||||||
|
return {'items': items, 'next_cursor': cursor}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _encode_cursor(resource: str, owner_id: str, created_at: str, item_id: str) -> str:
|
||||||
|
payload = {
|
||||||
|
"resource": resource,
|
||||||
|
"owner_id": owner_id,
|
||||||
|
"created_at": created_at,
|
||||||
|
"id": item_id,
|
||||||
|
}
|
||||||
|
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _decode_cursor(cursor: str, resource: str, owner_id: str) -> Tuple[str, str]:
|
||||||
|
try:
|
||||||
|
padded = cursor + "=" * (-len(cursor) % 4)
|
||||||
|
payload = json.loads(base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8"))
|
||||||
|
except (ValueError, UnicodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor") from exc
|
||||||
|
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor")
|
||||||
|
|
||||||
|
if payload.get("resource") != resource or payload.get("owner_id") != owner_id:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor")
|
||||||
|
|
||||||
|
item_id = ensure_uuid(payload.get("id"))
|
||||||
|
created_at = payload.get("created_at")
|
||||||
|
if not isinstance(created_at, str) or not created_at:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor")
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
||||||
|
if not created_at.endswith("Z") or parsed.tzinfo != timezone.utc:
|
||||||
|
raise ValueError("Expected UTC timestamp")
|
||||||
|
created_at = parsed.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor") from exc
|
||||||
|
return created_at, item_id
|
||||||
|
|
||||||
|
|
||||||
|
store = PostgresStore()
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
"""Attachment negative cases. Run against the Compose API at localhost:8080."""
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||||
|
from client import Client, check
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
client = Client()
|
||||||
|
alice, bob, eve, chat = client.bootstrap()
|
||||||
|
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png'
|
||||||
|
image = (Path(__file__).resolve().parents[1] / 'tests' / 'fixtures' / 'sample.png').read_bytes()
|
||||||
|
|
||||||
|
client.call('POST', path, 415, actor=alice, raw=b'not an image',
|
||||||
|
headers={'Content-Type': 'image/png'})
|
||||||
|
client.call('POST', path, 415, actor=alice, raw=image,
|
||||||
|
headers={'Content-Type': 'image/jpeg'})
|
||||||
|
client.call('POST', path, 413, actor=alice, raw=b'x' * (10 * 1024 * 1024 + 1),
|
||||||
|
headers={'Content-Type': 'image/png'})
|
||||||
|
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=../sample.png',
|
||||||
|
400, actor=alice, raw=image, headers={'Content-Type': 'image/png'})
|
||||||
|
|
||||||
|
attachment = client.call('POST', path, 201, actor=alice, raw=image,
|
||||||
|
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
|
||||||
|
client.call('GET', '/api/v1/attachments/' + attachment['id'], 404, actor=eve)
|
||||||
|
client.call('GET', '/api/v1/attachments/' + str(uuid4()), 404, actor=alice)
|
||||||
|
|
||||||
|
other = client.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||||
|
body={'title': 'Other chat', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
||||||
|
client.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice,
|
||||||
|
body={'text': '', 'attachment_ids': [attachment['id']]},
|
||||||
|
headers={'Idempotency-Key': uuid4().hex})
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
print('PASS: attachment validation and access checks')
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""Database atomicity and concurrent conflict checks, using isolated synthetic data."""
|
||||||
|
import unittest
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from unittest.mock import patch
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from src.database import create_db_engine
|
||||||
|
from src.errors import APIError
|
||||||
|
from src.models import Chat, IdempotencyRecord, Message
|
||||||
|
from src.store import PostgresStore
|
||||||
|
|
||||||
|
|
||||||
|
class IdempotencyTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.engine = create_db_engine()
|
||||||
|
self.store = PostgresStore()
|
||||||
|
self.store.sessions = sessionmaker(self.engine, expire_on_commit=False)
|
||||||
|
self.user = self.store.create_user('idem_' + uuid4().hex[:16], 'Idempotency', 'Synthetic-password-123')
|
||||||
|
self.chat = self.store.create_chat(self.user['id'], 'Atomic', [self.user['id']])
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_record_and_message_rollback_together(self):
|
||||||
|
real_add = Session.add
|
||||||
|
def fail_record(session, instance, *args, **kwargs):
|
||||||
|
if isinstance(instance, IdempotencyRecord):
|
||||||
|
raise RuntimeError('Simulate failure after message INSERT, before COMMIT')
|
||||||
|
return real_add(session, instance, *args, **kwargs)
|
||||||
|
args = (self.chat['id'], self.user['id'], 'atomic', 'same-key')
|
||||||
|
with patch.object(Session, 'add', fail_record):
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
self.store.create_message(*args)
|
||||||
|
with Session(self.engine) as session:
|
||||||
|
self.assertEqual(session.scalar(select(func.count()).select_from(Message).where(Message.chat_id == self.chat['id'])), 0)
|
||||||
|
self.assertIsNone(session.get(IdempotencyRecord, (self.user['id'], self.chat['id'], 'same-key')))
|
||||||
|
self.assertEqual(session.get(Chat, self.chat['id']).history_version, 0)
|
||||||
|
first = self.store.create_message(*args)
|
||||||
|
self.assertEqual(self.store.create_message(*args), first)
|
||||||
|
|
||||||
|
def test_concurrent_different_payloads_have_one_winner(self):
|
||||||
|
def send(index):
|
||||||
|
try:
|
||||||
|
return self.store.create_message(self.chat['id'], self.user['id'], str(index), 'conflict'), 201
|
||||||
|
except APIError as exc:
|
||||||
|
return None, exc.status_code
|
||||||
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
||||||
|
results = list(executor.map(send, range(8)))
|
||||||
|
self.assertEqual(sorted(code for _, code in results), [201] + [409] * 7)
|
||||||
|
with Session(self.engine) as session:
|
||||||
|
self.assertEqual(session.scalar(select(func.count()).select_from(Message).where(Message.chat_id == self.chat['id'])), 1)
|
||||||
|
self.assertEqual(session.get(Chat, self.chat['id']).history_version, 1)
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
"""Run inside the API container: python -m unittest discover -s student-tests -v."""
|
||||||
|
import base64
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||||
|
from client import Client
|
||||||
|
from src.auth import sessions
|
||||||
|
from src.database import create_db_engine, migrate
|
||||||
|
from src.models import User, Chat, ChatMember, SchemaMigration
|
||||||
|
|
||||||
|
|
||||||
|
class Lab2Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.engine = create_db_engine()
|
||||||
|
cls.sessions = sessionmaker(cls.engine)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
cls.engine.dispose()
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.client = Client('http://127.0.0.1:8080')
|
||||||
|
|
||||||
|
def test_registration_race(self):
|
||||||
|
body = {'username': 'race_' + uuid4().hex[:12], 'display_name': 'Race',
|
||||||
|
'password': 'Synthetic-password-123'}
|
||||||
|
def register(_):
|
||||||
|
return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status']
|
||||||
|
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||||
|
codes = list(executor.map(register, range(4)))
|
||||||
|
self.assertEqual(sorted(codes), [201, 409, 409, 409])
|
||||||
|
with self.sessions() as session:
|
||||||
|
count = session.scalar(select(func.count()).select_from(User).where(User.username == body['username']))
|
||||||
|
self.assertEqual(count, 1)
|
||||||
|
|
||||||
|
def test_chat_rollback(self):
|
||||||
|
actor = self.client.user('rollback')
|
||||||
|
title = 'rollback_' + uuid4().hex
|
||||||
|
self.client.call('POST', '/api/v1/chats', 400, actor=actor,
|
||||||
|
body={'title': title, 'member_ids': [actor['id'], str(uuid4())]})
|
||||||
|
with self.sessions() as session:
|
||||||
|
self.assertEqual(session.scalar(select(func.count()).select_from(Chat).where(Chat.title == title)), 0)
|
||||||
|
self.assertEqual(session.scalar(select(func.count()).select_from(ChatMember).where(
|
||||||
|
ChatMember.user_id == actor['id'])), 0)
|
||||||
|
|
||||||
|
def test_password_logout_and_identity(self):
|
||||||
|
actor = self.client.user('auth')
|
||||||
|
credential = base64.b64encode((actor['user']['username'] + ':Wrong-password-123').encode()).decode()
|
||||||
|
self.client.call('POST', '/api/v1/auth/sessions', 401,
|
||||||
|
headers={'Authorization': 'Basic ' + credential})
|
||||||
|
self.client.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Basic ' + credential})
|
||||||
|
self.client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']})
|
||||||
|
me = self.client.call('GET', '/api/v1/users/me', actor=actor,
|
||||||
|
headers={'X-User-Id': str(uuid4())}, schema='User')['json']
|
||||||
|
self.assertEqual(me['id'], actor['id'])
|
||||||
|
ttl = sessions.client.ttl(sessions.key(actor['token']))
|
||||||
|
self.assertGreater(ttl, 0)
|
||||||
|
self.assertLessEqual(ttl, 3600)
|
||||||
|
self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor)
|
||||||
|
self.client.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||||
|
self.client.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=actor)
|
||||||
|
self.assertIsNone(sessions.client.get(sessions.key(actor['token'])))
|
||||||
|
with self.sessions() as session:
|
||||||
|
encoded = session.get(User, actor['id']).password_hash
|
||||||
|
self.assertTrue(encoded.startswith('$argon2id$'))
|
||||||
|
self.assertNotIn(actor['registration']['password'], encoded)
|
||||||
|
|
||||||
|
def test_unicode_password(self):
|
||||||
|
username = 'unicode_' + uuid4().hex[:12]
|
||||||
|
password = '🙂:я' * 40
|
||||||
|
self.client.call('POST', '/api/v1/users', 201,
|
||||||
|
body={'username': username, 'display_name': 'Unicode', 'password': password})
|
||||||
|
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||||
|
result = self.client.call('POST', '/api/v1/auth/sessions', 201,
|
||||||
|
headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
|
||||||
|
self.assertEqual(result['user']['username'], username)
|
||||||
|
|
||||||
|
def test_migrations_repeat(self):
|
||||||
|
with self.sessions() as session:
|
||||||
|
before = session.scalar(select(func.count()).select_from(User))
|
||||||
|
migrate(self.engine)
|
||||||
|
migrate(self.engine)
|
||||||
|
with self.sessions() as session:
|
||||||
|
self.assertEqual(session.scalar(select(func.count()).select_from(User)), before)
|
||||||
|
versions = set(session.scalars(select(SchemaMigration.version)))
|
||||||
|
expected = {path.name for path in (Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')}
|
||||||
|
self.assertEqual(versions, expected)
|
||||||
|
|
||||||
|
def test_expiry_short_ttl(self):
|
||||||
|
process = subprocess.Popen(
|
||||||
|
[sys.executable, '-m', 'src'],
|
||||||
|
env={**os.environ, 'SESSION_TTL_SECONDS': '2', 'APP_PORT': '8081', 'APP_HOST': '127.0.0.1'},
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
client = Client('http://127.0.0.1:8081')
|
||||||
|
for _ in range(100):
|
||||||
|
try:
|
||||||
|
client.call('GET', '/health/ready')
|
||||||
|
break
|
||||||
|
except AssertionError:
|
||||||
|
self.assertIsNone(process.poll(), 'Short-TTL API exited')
|
||||||
|
time.sleep(0.1)
|
||||||
|
else:
|
||||||
|
self.fail('Short-TTL API did not become ready')
|
||||||
|
actor = client.user('expiry')
|
||||||
|
client.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||||
|
time.sleep(2.2)
|
||||||
|
client.call('GET', '/api/v1/users/me', 401, actor=actor)
|
||||||
|
self.assertIsNone(sessions.client.get(sessions.key(actor['token'])))
|
||||||
|
finally:
|
||||||
|
process.terminate()
|
||||||
|
process.wait(timeout=10)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user