ЛР 4 — БВТ2301 — Гамаюнов — на 3 #163
+3
-18
@@ -1,20 +1,5 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Local Lab 4 Compose settings. The init script stores credentials separately.
|
||||
# Lab 3 on the same machine can keep 8080. Use 8080 here if it is free.
|
||||
LAB4_PORT=8084
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
*.sh text eol=lf
|
||||
*.py text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.ps1 text eol=lf
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
FROM python:3.12.10-slim-bookworm
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080
|
||||
WORKDIR /app
|
||||
COPY requirements.txt requirements.lock ./
|
||||
RUN pip install --no-cache-dir -r requirements.lock && useradd --create-home --uid 10001 appuser
|
||||
COPY --chown=appuser:appuser src ./src
|
||||
COPY --chown=appuser:appuser migrations ./migrations
|
||||
USER appuser
|
||||
EXPOSE 8080
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=3s --timeout=3s --start-period=10s --retries=5 CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).read()" || exit 1
|
||||
CMD ["sh", "-c", "exec uvicorn src.main:app --host \"$APP_HOST\" --port \"$APP_PORT\" --no-access-log --timeout-graceful-shutdown 3"]
|
||||
@@ -1,37 +1,83 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа — Гамаюнов Артём, БВТ2301.
|
||||
- Целевая оценка — **3**.
|
||||
- Коммит и PR оформляются пользователем после независимого ревью.
|
||||
- Источники и помощь ИИ — нормативные `contracts/README.md`, `contracts/openapi.json` и публичные тесты этого репозитория, собственная ЛР3 (`be96c39`, `96ad3da`). Codex помог перенести приложение, реализовать вложения, провести локальные Docker-проверки и оформить материалы. [RustFS в контейнере](https://docs.rustfs.com/en/installation/container), [настройки Boto3](https://docs.aws.amazon.com/boto3/latest/guide/configuration.html).
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Откройте корень репозитория в PowerShell при запущенном Docker Desktop в режиме Linux containers. Порт по умолчанию — `127.0.0.1:8084`, поскольку на машине проверки 8080 занят ЛР3. При свободном 8080 можно задать `LAB4_PORT=8080` в локальном `.env`.
|
||||
|
||||
```powershell
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\init-env.ps1
|
||||
docker compose --project-name mtusi-lab4-grade3 config --quiet
|
||||
docker compose --project-name mtusi-lab4-grade3 pull postgres lb s3
|
||||
docker compose --project-name mtusi-lab4-grade3 --profile test build migrate tools
|
||||
docker compose --project-name mtusi-lab4-grade3 up -d --no-build --wait --wait-timeout 70
|
||||
curl.exe --fail --retry 10 --retry-connrefused http://localhost:8084/health/ready
|
||||
```
|
||||
|
||||
`init-env.ps1` создаёт `.env` из `.env.example`, случайные секреты в `secrets/` и ограничивает ACL каталога текущим пользователем и SYSTEM. Повтор сохраняет значения. Compose монтирует секреты из файлов. `migrate` применяет SQL-версии под транзакционной блокировкой. `bootstrap` подписанным S3-запросом проверяет или создаёт приватный бакет и удаляет ранее добавленную bucket policy. Оба шага должны завершиться с кодом 0 до запуска API. RustFS использует файловые параметры credentials, поэтому его entrypoint сбрасывает переменные с заводскими значениями образа.
|
||||
|
||||
```powershell
|
||||
docker compose --project-name mtusi-lab4-grade3 --profile test run --rm --no-deps tools make check validate test
|
||||
docker compose --project-name mtusi-lab4-grade3 --profile test run --rm --no-deps tools make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
docker compose --project-name mtusi-lab4-grade3 --profile test run --rm --no-deps tools make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose --project-name mtusi-lab4-grade3 start api1 api2
|
||||
docker compose --project-name mtusi-lab4-grade3 --profile test run --rm --no-deps tools python student-tests/grade3.py
|
||||
```
|
||||
|
||||
В контейнере `tools` адрес балансировщика — `http://lb:8080`. После публичного failover остановленная реплика запускается явно. Студенческий сценарий ждёт обе реплики и сам восстанавливает сервисы после своих проверок. Docker socket подключён только в тестовом профиле `tools`. Это привилегированный локальный инструмент для остановки, перезапуска и Docker Inspect. В образе hooks имеют режим 0755 независимо от Windows checkout.
|
||||
|
||||
Для безопасной остановки используйте `docker compose --project-name mtusi-lab4-grade3 stop`. Для намеренного удаления всех учебных данных **только ЛР4** используйте `docker compose --project-name mtusi-lab4-grade3 down --volumes`. PostgreSQL и S3 хранятся в двух именованных volumes этого проекта.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Клиент → HAProxy → `api1` / `api2` → общий PostgreSQL и приватный RustFS. Из ЛР3 перенесены модули `src/`, миграции 001/002, сборка FastAPI, Compose, HAProxy и Linux test tooling. Сохранены пользователи, чаты, членство, Argon2id, Basic при входе, отзыв и TTL Bearer-сессий, сортировка и keyset pagination, заголовки `X-Request-Id` и `X-Instance-Id`. Объём прав API определяется членством во время запроса.
|
||||
|
||||
| Роль | Сервис и версия | Готовность, сеть и данные |
|
||||
| --- | --- | --- |
|
||||
| Публичный вход | `lb`, HAProxy 3.0.11 Alpine | Round robin с `GET /health/ready`; единственная публикация на loopback |
|
||||
| HTTP | `api1`, `api2`, Python 3.12.10, FastAPI 0.115.12, Uvicorn 0.34.2 | Внутренний 8080; PostgreSQL и подписанный HEAD бакета для readiness; без собственных volumes |
|
||||
| Схема и метаданные | `migrate` и `postgres`, PostgreSQL 16.8 | `pg_isready`, внутренний 5432, `postgres_data`, файловые DB credentials |
|
||||
| Бинарные оригиналы | `bootstrap` и `s3`, RustFS 1.0.0-alpha.85 с digest в Compose | `GET /health`, внутренний 9000, `s3_data`, консоль отключена, файловые S3 credentials |
|
||||
| Проверки | `tools`, Docker CLI 27.5.1, Linux Python и make | Внутренняя сеть, Docker socket только для локальных тестов |
|
||||
|
||||
Выбран реально загруженный RustFS. Docker Hub отклонил запрос на выбранный тег MinIO. Runtime- и test-зависимости закреплены в `requirements.lock` и `infra/requirements-tools.lock`; для вложений используются Pillow 12.2.0 и Boto3 1.40.45. Из `.env.example` читаются `LAB4_PORT`, `SESSION_TTL_SECONDS` (3600 по умолчанию) и `S3_BUCKET`. Адреса, регион, пути смонтированных секретов и две реплики заданы в Compose. Лимиты 10 MiB, 40 млн пикселей и TTL ключа 24 часа фиксированы контрактом.
|
||||
|
||||
Миграция `003_attachments.sql` добавляет `attachments` (владелец, чат, имя, MIME, размер, SHA-256, состояние, ключ объекта, timestamps) и `message_attachments` (упорядоченные ссылки). Составные внешние ключи гарантируют один чат у вложения и сообщения. У старых сообщений массив вложений вычисляется как `[]`, их данные и записи идемпотентности сохраняются. Новый POST сообщения резервирует ключ, проверяет принадлежность и готовность вложений, пишет сообщение и связи в одной транзакции. При повторе сравниваются текст и нормализованный упорядоченный список ID. Незаполненный список эквивалентен пустому. Ошибка транзакции откатывает резервирование. История сортируется по `(created_at, id)`. Между страницами нет snapshot, поэтому более ранний timestamp параллельной транзакции может оказаться позади уже выданного курсора.
|
||||
|
||||
Три маршрута — POST `/api/v1/chats/{chat_id}/attachments?filename=…`, GET `/api/v1/attachments/{id}` и GET `/api/v1/attachments/{id}/content`. POST проверяет участника, MIME и отображаемое имя, ограничивает чтение тела, проверяет структуру и декодирует изображение после проверки пикселей. Случайный ключ S3 генерируется независимо от имени и ID вложения. API отправляет исходные байты в S3, затем фиксирует метаданные и только после commit возвращает 201. GET content проверяет ACL, длину и SHA-256 и передаёт байты через API. Объект и метаданные лежат в разных хранилищах.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Структура и схемы | `make check validate` | Обе проверки PASS | [checks.txt](evidence/grade3/checks.txt) |
|
||||
| Публичный smoke | `make test`, профиль `lab.json` №4 | PASS, в том числе авторизация, история, идемпотентность, checksum, ACL и logout | [checks.txt](evidence/grade3/checks.txt) |
|
||||
| Общий стенд и private bucket | `docker compose up -d --no-build --wait`; внутренний анонимный GET к существующему объекту | Два API отвечают 200, миграции и bootstrap завершились с кодом 0, S3 GET без credentials — 403, порт S3 не опубликован | [environment.txt](evidence/grade3/environment.txt), [student.txt](evidence/grade3/student.txt) |
|
||||
| Синхронный upload/download, лимиты | `python student-tests/grade3.py` | PNG 72 B и JPEG 635 B — 201 ready, GET 200, идентичные байты и SHA-256. Ошибки filename 400, данных/MIME 415, 10 MiB+1 с/без Content-Length и 40 005 000 px — 413 | [student.txt](evidence/grade3/student.txt) |
|
||||
| ACL, история и идемпотентность с вложениями | Тот же сценарий | Посторонний 404, без сессии 401, другой чат и неизвестный ID 404, недоступное вложение 409. Два конкурентных POST через api1/api2 вернули одно Message, другой список — 409, отсутствие списка равно `[]` | [student.txt](evidence/grade3/student.txt) |
|
||||
| Сохранность | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` и сверка Docker Inspect | Тот же Bearer, история, метаданные и байты. Изменился `StartedAt` PostgreSQL, S3 и обоих API при тех же container ID | [persistence.txt](evidence/grade3/persistence.txt), [student.txt](evidence/grade3/student.txt) |
|
||||
| Переключение API | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`; затем оба пути upload → stop → download в студенческом тесте | Публичная запись восстановилась за 1,434 с; исходные байты получены через выжившую реплику для обоих направлений | [failover.txt](evidence/grade3/failover.txt), [student.txt](evidence/grade3/student.txt) |
|
||||
| Временный отказ S3 | Студенческий тест останавливает реальный контейнер S3 и запускает его снова | Через HAProxy upload/download вернули 503. Напрямую API вернул JSON 503 для upload/download/readiness и 200 для liveness. После запуска файл снова доступен | [student.txt](evidence/grade3/student.txt) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Финальная проверка выполнена в отдельном проекте `mtusi-lab4-grade3-review` на loopback-порту 8085 с новыми volumes. Docker Engine 29.9.3 (Linux/amd64), Compose 5.7.0, Docker VM с 4 CPU и 8 242 380 800 байт RAM. Сценарии функциональные. Бюджет failover — 10 секунд. Время студенческого сценария отсчитывается от запуска hook до первого успешного скачивания после остановки, включая проверку Docker Inspect. Публичный тест измеряет первую запись после завершения hook.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Остановить загрузившую `api1` | `api2` отдаёт оригинал | 201 → 200, остановленный контейнер подтверждён через Docker Inspect | 1,207 с, совпали байты и SHA-256 |
|
||||
| Остановить загрузившую `api2` | `api1` отдаёт оригинал | 201 → 200, остановленный контейнер подтверждён через Docker Inspect | 1,186 с, совпали байты и SHA-256 |
|
||||
| Перезапустить PostgreSQL, S3, api1 и api2 | Уцелели сессия, сообщения и изображения | Docker Inspect подтвердил новый `StartedAt`; метаданные и оригинал получены со старым Bearer | Подтверждённые данные сохранены |
|
||||
| Остановить и запустить S3 | Контролируемый 503 во время отказа | Реальная остановка, 503, затем readiness 200 и успешное скачивание | Оригинал и метаданные сохранены |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
S3 и PostgreSQL не разделяют транзакцию. Если после успешного PUT запись метаданных точно сорвалась, приложение пробует удалить только созданный объект. Падение процесса, таймаут PUT, сбой удаления или неопределённый исход commit могут оставить объект без метаданных. При неопределённом commit объект сохраняется, чтобы подтверждённые метаданные не указывали на удалённые байты. Автоматическая сверка не входит в этот вариант.
|
||||
|
||||
Учебный RustFS одноузловой, его административная учётная запись также используется API. Публичным входом остаётся только локальный HAProxy. HAProxy, PostgreSQL, RustFS и Docker-host — единичные точки отказа. Функциональный failover проверяет потерю одной API-реплики. Проверка readiness использует HEAD бакета и SQL SELECT. Когда S3 недоступен и HAProxy исключил обе реплики, балансировщик сам выдаёт 503.
|
||||
|
||||
Загрузка ограничена двумя одновременными декодированиями на реплику и 10 MiB входных байтов на запрос. Декодированное изображение и буфер скачивания расходуют память процесса. Пропускная способность при параллельной работе с большими файлами не измерялась.
|
||||
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
name: mtusi-lab4-grade3
|
||||
|
||||
x-s3-env: &s3-env
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
|
||||
x-app: &app
|
||||
image: mtusi-lab4-api:grade3
|
||||
pull_policy: never
|
||||
|
||||
x-api: &api
|
||||
<<: *app
|
||||
environment: &api-env
|
||||
<<: *s3-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
bootstrap: {condition: service_completed_successfully}
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 6s
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.8-bookworm
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: rustfs/rustfs:1.0.0-alpha.85@sha256:23ff2af1ec2d64e0fb9608a1c39b5eb96fcff3a8efecf332986c948b0e3e4f8a
|
||||
entrypoint: [sh, -ec, "unset RUSTFS_ACCESS_KEY RUSTFS_SECRET_KEY; exec /entrypoint.sh rustfs"]
|
||||
environment:
|
||||
RUSTFS_VOLUMES: /data
|
||||
RUSTFS_ADDRESS: ":9000"
|
||||
RUSTFS_CONSOLE_ENABLE: "false"
|
||||
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_access_key_id
|
||||
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [s3_access_key_id, s3_secret_access_key]
|
||||
volumes: [s3_data:/data]
|
||||
healthcheck:
|
||||
test: [CMD, curl, -fsS, --max-time, "2", "http://127.0.0.1:9000/health"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
<<: *app
|
||||
build: .
|
||||
command: [python, -m, src.db]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
healthcheck: {disable: true}
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
|
||||
bootstrap:
|
||||
<<: *app
|
||||
command: [python, -m, src.s3]
|
||||
environment: *s3-env
|
||||
secrets: [s3_access_key_id, s3_secret_access_key]
|
||||
healthcheck: {disable: true}
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
|
||||
api1:
|
||||
<<: *api
|
||||
environment:
|
||||
<<: *api-env
|
||||
INSTANCE_ID: api1
|
||||
|
||||
api2:
|
||||
<<: *api
|
||||
environment:
|
||||
<<: *api-env
|
||||
INSTANCE_ID: api2
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.11-alpine
|
||||
ports: ["127.0.0.1:${LAB4_PORT:-8084}:8080"]
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
restart: unless-stopped
|
||||
|
||||
tools:
|
||||
profiles: [test]
|
||||
image: mtusi-lab4-tools:grade3
|
||||
build: {context: ., dockerfile: infra/Dockerfile.tools}
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
BASE_URL: http://lb:8080
|
||||
LAB4_PROJECT: ${COMPOSE_PROJECT_NAME}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
|
||||
secrets:
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
s3_access_key_id: {file: ./secrets/s3_access_key_id}
|
||||
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
s3_data:
|
||||
@@ -0,0 +1,8 @@
|
||||
Container mtusi-lab4-grade3-review-tools-run-4213a26d98ae Creating
|
||||
Container mtusi-lab4-grade3-review-tools-run-4213a26d98ae Created
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://lb:8080"
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,85 @@
|
||||
Fresh Compose project mtusi-lab4-grade3-review, host port 8085, no preexisting volumes.
|
||||
Docker Engine 29.9.3 linux/amd64
|
||||
Docker Compose version v5.7.0
|
||||
Docker VM CPUs=4 memory_bytes=8242380800
|
||||
compose config --quiet: exit 0
|
||||
Volume mtusi-lab4-grade3-review_s3_data Creating
|
||||
Volume mtusi-lab4-grade3-review_s3_data Creating
|
||||
Volume mtusi-lab4-grade3-review_postgres_data Creating
|
||||
Volume mtusi-lab4-grade3-review_postgres_data Creating
|
||||
Network mtusi-lab4-grade3-review_default Creating
|
||||
Network mtusi-lab4-grade3-review_default Creating
|
||||
Volume mtusi-lab4-grade3-review_s3_data Created
|
||||
Volume mtusi-lab4-grade3-review_s3_data Created
|
||||
Volume mtusi-lab4-grade3-review_postgres_data Created
|
||||
Volume mtusi-lab4-grade3-review_postgres_data Created
|
||||
Network mtusi-lab4-grade3-review_default Created
|
||||
Network mtusi-lab4-grade3-review_default Created
|
||||
Container mtusi-lab4-grade3-review-lb-1 Creating
|
||||
Container mtusi-lab4-grade3-review-postgres-1 Creating
|
||||
Container mtusi-lab4-grade3-review-s3-1 Creating
|
||||
Container mtusi-lab4-grade3-review-s3-1 Created
|
||||
Container mtusi-lab4-grade3-review-bootstrap-1 Creating
|
||||
Container mtusi-lab4-grade3-review-lb-1 Created
|
||||
Container mtusi-lab4-grade3-review-postgres-1 Created
|
||||
Container mtusi-lab4-grade3-review-migrate-1 Creating
|
||||
Container mtusi-lab4-grade3-review-migrate-1 Created
|
||||
Container mtusi-lab4-grade3-review-bootstrap-1 Created
|
||||
Container mtusi-lab4-grade3-review-api2-1 Creating
|
||||
Container mtusi-lab4-grade3-review-api1-1 Creating
|
||||
Container mtusi-lab4-grade3-review-api2-1 Created
|
||||
Container mtusi-lab4-grade3-review-api1-1 Created
|
||||
Container mtusi-lab4-grade3-review-s3-1 Starting
|
||||
Container mtusi-lab4-grade3-review-lb-1 Starting
|
||||
Container mtusi-lab4-grade3-review-postgres-1 Starting
|
||||
Container mtusi-lab4-grade3-review-s3-1 Started
|
||||
Container mtusi-lab4-grade3-review-s3-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-lb-1 Started
|
||||
Container mtusi-lab4-grade3-review-postgres-1 Started
|
||||
Container mtusi-lab4-grade3-review-postgres-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-s3-1 Healthy
|
||||
Container mtusi-lab4-grade3-review-bootstrap-1 Starting
|
||||
Container mtusi-lab4-grade3-review-bootstrap-1 Started
|
||||
Container mtusi-lab4-grade3-review-bootstrap-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-postgres-1 Healthy
|
||||
Container mtusi-lab4-grade3-review-migrate-1 Starting
|
||||
Container mtusi-lab4-grade3-review-bootstrap-1 Exited
|
||||
Container mtusi-lab4-grade3-review-migrate-1 Started
|
||||
Container mtusi-lab4-grade3-review-migrate-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-migrate-1 Exited
|
||||
Container mtusi-lab4-grade3-review-api2-1 Starting
|
||||
Container mtusi-lab4-grade3-review-api1-1 Starting
|
||||
Container mtusi-lab4-grade3-review-api2-1 Started
|
||||
Container mtusi-lab4-grade3-review-api1-1 Started
|
||||
Container mtusi-lab4-grade3-review-migrate-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-postgres-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-bootstrap-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-lb-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-s3-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-api1-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-api2-1 Waiting
|
||||
Container mtusi-lab4-grade3-review-bootstrap-1 Exited
|
||||
Container mtusi-lab4-grade3-review-migrate-1 Exited
|
||||
Container mtusi-lab4-grade3-review-lb-1 Healthy
|
||||
Container mtusi-lab4-grade3-review-postgres-1 Healthy
|
||||
Container mtusi-lab4-grade3-review-s3-1 Healthy
|
||||
Container mtusi-lab4-grade3-review-api2-1 Healthy
|
||||
Container mtusi-lab4-grade3-review-api1-1 Healthy
|
||||
migrate-1 | PASS: versioned migrations applied (already applied versions skipped)
|
||||
bootstrap-1 | PASS: private attachment bucket bootstrapped
|
||||
api1 Up 7 seconds (healthy)
|
||||
api2 Up 7 seconds (healthy)
|
||||
bootstrap Exited (0) 10 seconds ago
|
||||
lb Up 16 seconds
|
||||
migrate Exited (0) 8 seconds ago
|
||||
postgres Up 15 seconds (healthy)
|
||||
s3 Up 16 seconds (healthy)
|
||||
mtusi-lab4-grade3-review_postgres_data
|
||||
mtusi-lab4-grade3-review_s3_data
|
||||
rustfs/rustfs:1.0.0-alpha.85 sha256:23ff2af1ec2d64e0fb9608a1c39b5eb96fcff3a8efecf332986c948b0e3e4f8a rustfs/rustfs@sha256:23ff2af1ec2d64e0fb9608a1c39b5eb96fcff3a8efecf332986c948b0e3e4f8a
|
||||
postgres:16.8-bookworm sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae postgres@sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae
|
||||
haproxy:3.0.11-alpine sha256:e49b7beb96f4dba79f9a671515b469a6e516a2b88ff0681a820b184bba1effbd haproxy@sha256:e49b7beb96f4dba79f9a671515b469a6e516a2b88ff0681a820b184bba1effbd
|
||||
python:3.12.10-slim-bookworm sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db python@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db
|
||||
docker:27.5.1-cli sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c docker@sha256:851f91d241214e7c6db86513b270d58776379aacc5eb9c4a87e5b47115e3065c
|
||||
mtusi-lab4-api:grade3 sha256:368680a5d1ed346e32407cd7e0825f64bd4bfdf38978416b652b59ac554a73bd mtusi-lab4-api@sha256:368680a5d1ed346e32407cd7e0825f64bd4bfdf38978416b652b59ac554a73bd
|
||||
mtusi-lab4-tools:grade3 sha256:cccf094afef55e8798a685df5dd458910f89a5cc5a09102ec138dad22b34acb1 mtusi-lab4-tools@sha256:cccf094afef55e8798a685df5dd458910f89a5cc5a09102ec138dad22b34acb1
|
||||
@@ -0,0 +1,18 @@
|
||||
Before hook:
|
||||
postgres 7cd237dd8fa6cd06a6f39612d9a41a32b6480068d71e62c628f18fccfc087ec8 running=true StartedAt=2026-11-20T14:25:02.815830365Z FinishedAt=2026-11-20T14:25:02.562801189Z
|
||||
s3 5dd4667ff46ee928a659fd29bda22b2a1439b011b3c193906adc9804f7e4ed51 running=true StartedAt=2026-11-20T14:25:04.898321973Z FinishedAt=2026-11-20T14:25:04.671438688Z
|
||||
api1 9ddefeacc1bf4972ed0f520e5697853fdab3f5f9a096b7119477572c4fb579a3 running=true StartedAt=2026-11-20T14:25:06.79818885Z FinishedAt=2026-11-20T14:25:06.540284457Z
|
||||
api2 6c76d2b6d600b56aef6e11629f4235be25f35c4a8ce45563f8fedf71e2f4421b running=true StartedAt=2026-11-20T14:25:08.041189785Z FinishedAt=2026-11-20T14:25:07.758803343Z
|
||||
lb 098788c707886ccb5273d6a7ee07e82c56444824c4124828854ed095d94104ff running=true StartedAt=2026-11-20T14:23:48.23848068Z FinishedAt=0001-01-01T00:00:00Z
|
||||
Container mtusi-lab4-grade3-review-tools-run-84fd9402cde6 Creating
|
||||
Container mtusi-lab4-grade3-review-tools-run-84fd9402cde6 Created
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 117, "failed_attempts": 0, "first_recovery_seconds": 1.434}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
After hook:
|
||||
postgres 7cd237dd8fa6cd06a6f39612d9a41a32b6480068d71e62c628f18fccfc087ec8 running=true StartedAt=2026-11-20T14:25:02.815830365Z FinishedAt=2026-11-20T14:25:02.562801189Z
|
||||
s3 5dd4667ff46ee928a659fd29bda22b2a1439b011b3c193906adc9804f7e4ed51 running=true StartedAt=2026-11-20T14:25:04.898321973Z FinishedAt=2026-11-20T14:25:04.671438688Z
|
||||
api1 9ddefeacc1bf4972ed0f520e5697853fdab3f5f9a096b7119477572c4fb579a3 running=false StartedAt=2026-11-20T14:25:06.79818885Z FinishedAt=2026-11-20T14:25:24.443528331Z
|
||||
api2 6c76d2b6d600b56aef6e11629f4235be25f35c4a8ce45563f8fedf71e2f4421b running=true StartedAt=2026-11-20T14:25:08.041189785Z FinishedAt=2026-11-20T14:25:07.758803343Z
|
||||
lb 098788c707886ccb5273d6a7ee07e82c56444824c4124828854ed095d94104ff running=true StartedAt=2026-11-20T14:23:48.23848068Z FinishedAt=0001-01-01T00:00:00Z
|
||||
@@ -0,0 +1,17 @@
|
||||
Before hook:
|
||||
postgres 7cd237dd8fa6cd06a6f39612d9a41a32b6480068d71e62c628f18fccfc087ec8 running=true StartedAt=2026-11-20T14:23:49.020172233Z FinishedAt=0001-01-01T00:00:00Z
|
||||
s3 5dd4667ff46ee928a659fd29bda22b2a1439b011b3c193906adc9804f7e4ed51 running=true StartedAt=2026-11-20T14:23:47.65345572Z FinishedAt=0001-01-01T00:00:00Z
|
||||
api1 9ddefeacc1bf4972ed0f520e5697853fdab3f5f9a096b7119477572c4fb579a3 running=true StartedAt=2026-11-20T14:23:56.918273399Z FinishedAt=0001-01-01T00:00:00Z
|
||||
api2 6c76d2b6d600b56aef6e11629f4235be25f35c4a8ce45563f8fedf71e2f4421b running=true StartedAt=2026-11-20T14:23:56.590364071Z FinishedAt=0001-01-01T00:00:00Z
|
||||
lb 098788c707886ccb5273d6a7ee07e82c56444824c4124828854ed095d94104ff running=true StartedAt=2026-11-20T14:23:48.23848068Z FinishedAt=0001-01-01T00:00:00Z
|
||||
Container mtusi-lab4-grade3-review-tools-run-8970ed66a2ba Creating
|
||||
Container mtusi-lab4-grade3-review-tools-run-8970ed66a2ba Created
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
After hook:
|
||||
postgres 7cd237dd8fa6cd06a6f39612d9a41a32b6480068d71e62c628f18fccfc087ec8 running=true StartedAt=2026-11-20T14:25:02.815830365Z FinishedAt=2026-11-20T14:25:02.562801189Z
|
||||
s3 5dd4667ff46ee928a659fd29bda22b2a1439b011b3c193906adc9804f7e4ed51 running=true StartedAt=2026-11-20T14:25:04.898321973Z FinishedAt=2026-11-20T14:25:04.671438688Z
|
||||
api1 9ddefeacc1bf4972ed0f520e5697853fdab3f5f9a096b7119477572c4fb579a3 running=true StartedAt=2026-11-20T14:25:06.79818885Z FinishedAt=2026-11-20T14:25:06.540284457Z
|
||||
api2 6c76d2b6d600b56aef6e11629f4235be25f35c4a8ce45563f8fedf71e2f4421b running=true StartedAt=2026-11-20T14:25:08.041189785Z FinishedAt=2026-11-20T14:25:07.758803343Z
|
||||
lb 098788c707886ccb5273d6a7ee07e82c56444824c4124828854ed095d94104ff running=true StartedAt=2026-11-20T14:23:48.23848068Z FinishedAt=0001-01-01T00:00:00Z
|
||||
@@ -0,0 +1,16 @@
|
||||
Container mtusi-lab4-grade3-review-tools-run-e0c01bb49ab4 Creating
|
||||
Container mtusi-lab4-grade3-review-tools-run-e0c01bb49ab4 Created
|
||||
PASS HAProxy readiness: api1=200 api2=200
|
||||
PASS image/png upload=201 ready download=200 bytes=72 sha256=1e0e7940b11ad9d314e82ad00c5aeda952d2a83747b85aeaec86d8af23006b9b
|
||||
PASS image/jpeg upload=201 ready download=200 bytes=635 sha256=fcece858d975d772f297f454b1b476751a5d956a9ecd5357720689a40e65d4cf
|
||||
PASS ACL metadata/content/upload: outsider=404 missing Bearer=401; filenames=400; damaged/false MIME=415; 10 MiB+1 with/without Content-Length=413; 40,005,000 pixels=413
|
||||
PASS anonymous internal GET of known existing object: 403; Docker publishes no S3 ports
|
||||
PASS concurrent attached POST: api1/api2=201 same message; one history row; changed IDs=409; omitted=[]; both cursor pages; foreign/unknown=404, failed=409, invalid lists=400
|
||||
PASS failover upload=api1 201 stopped=9ddefeacc1bf FinishedAt=2026-11-20T14:26:01.189358075Z download=api2 200 seconds=1.207 sha256=1e0e7940b11ad9d314e82ad00c5aeda952d2a83747b85aeaec86d8af23006b9b
|
||||
PASS failover upload=api2 201 stopped=6c76d2b6d600 FinishedAt=2026-11-20T14:26:06.566233653Z download=api1 200 seconds=1.186 sha256=1e0e7940b11ad9d314e82ad00c5aeda952d2a83747b85aeaec86d8af23006b9b
|
||||
PASS S3 stopped=5dd4667ff46e: LB upload/download=503; API JSON upload/download/ready=503 live=200
|
||||
restart postgres 7cd237dd8fa6 StartedAt=2026-11-20T14:25:02.815830365Z -> 2026-11-20T14:26:26.753366571Z
|
||||
restart s3 5dd4667ff46e StartedAt=2026-11-20T14:26:23.574434081Z -> 2026-11-20T14:26:28.918932541Z
|
||||
restart api1 9ddefeacc1bf StartedAt=2026-11-20T14:26:01.806050964Z -> 2026-11-20T14:26:33.051002536Z
|
||||
restart api2 6c76d2b6d600 StartedAt=2026-11-20T14:26:07.302613564Z -> 2026-11-20T14:26:34.211556495Z
|
||||
PASS original Bearer metadata/content=200 after restart sha256=1e0e7940b11ad9d314e82ad00c5aeda952d2a83747b85aeaec86d8af23006b9b
|
||||
@@ -0,0 +1,14 @@
|
||||
FROM docker:27.5.1-cli
|
||||
RUN apk add --no-cache python3 py3-pip make && python3 -m venv /opt/venv
|
||||
ENV PATH=/opt/venv/bin:$PATH PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
WORKDIR /workspace
|
||||
COPY infra/requirements-tools.lock /tmp/requirements-tools.lock
|
||||
RUN pip install --no-cache-dir -r /tmp/requirements-tools.lock
|
||||
COPY Makefile lab.json README.md COURSE.md CONTRIBUTING.md REPORT.md ./
|
||||
COPY contracts/ ./contracts/
|
||||
COPY tests/ ./tests/
|
||||
COPY scripts/check_contract.py scripts/validate_openapi.py ./scripts/
|
||||
COPY --chmod=0755 scripts/restart.sh scripts/stop-one.sh scripts/lab-containers.sh ./scripts/
|
||||
COPY student-tests/ ./student-tests/
|
||||
ENTRYPOINT []
|
||||
CMD ["make", "check"]
|
||||
@@ -0,0 +1,29 @@
|
||||
defaults
|
||||
mode http
|
||||
timeout http-request 5s
|
||||
timeout connect 500ms
|
||||
timeout client 15s
|
||||
timeout server 10s
|
||||
timeout check 1s
|
||||
retries 1
|
||||
retry-on conn-failure
|
||||
option redispatch
|
||||
|
||||
resolvers docker
|
||||
nameserver embedded 127.0.0.11:53
|
||||
timeout resolve 500ms
|
||||
timeout retry 500ms
|
||||
hold valid 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend public_http
|
||||
bind :8080
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 1 resolvers docker init-addr none resolve-prefer ipv4
|
||||
server api1 api1:8080
|
||||
server api2 api2:8080
|
||||
@@ -0,0 +1,20 @@
|
||||
attrs==26.1.0
|
||||
certifi==2026.10.23
|
||||
charset-normalizer==3.5.2
|
||||
idna==3.20
|
||||
jsonschema==4.26.0
|
||||
jsonschema-path==0.3.4
|
||||
jsonschema-specifications==2025.9.1
|
||||
lazy-object-proxy==1.13.0
|
||||
openapi-schema-validator==0.6.3
|
||||
openapi-spec-validator==0.7.2
|
||||
pathable==0.4.4
|
||||
PyYAML==6.0.3
|
||||
referencing==0.36.2
|
||||
requests==2.34.2
|
||||
rfc3339-validator==0.1.4
|
||||
rpds-py==2026.9.1
|
||||
six==1.17.0
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
Pillow==12.2.0
|
||||
@@ -0,0 +1,36 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100 AND btrim(display_name) <> ''),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100 AND btrim(title) <> ''),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user ON chat_members(user_id, chat_id);
|
||||
CREATE INDEX chats_order ON chats(created_at, id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000 AND btrim(text) <> ''),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX messages_history ON messages(chat_id, created_at, id);
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_expiry ON sessions(expires_at);
|
||||
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE message_idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
method text NOT NULL,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL CHECK (key COLLATE "C" ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
message_id uuid NOT NULL REFERENCES messages(id) DEFERRABLE INITIALLY DEFERRED,
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, method, chat_id, key)
|
||||
);
|
||||
@@ -0,0 +1,35 @@
|
||||
-- Preserve existing message rows and stored idempotency keys.
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
|
||||
ALTER TABLE messages ADD UNIQUE (chat_id, id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL,
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255
|
||||
AND strpos(filename, '/') = 0 AND strpos(filename, chr(92)) = 0
|
||||
AND filename !~ '[[:cntrl:]]'),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 COLLATE "C" ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
status text NOT NULL DEFAULT 'ready' CHECK (status IN ('ready', 'failed')),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT transaction_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT transaction_timestamp(),
|
||||
CHECK ((status = 'ready') = (error_code IS NULL)),
|
||||
FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id),
|
||||
UNIQUE (chat_id, id)
|
||||
);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
chat_id uuid NOT NULL,
|
||||
message_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position integer NOT NULL CHECK (position BETWEEN 1 AND 10),
|
||||
PRIMARY KEY (message_id, position),
|
||||
UNIQUE (message_id, attachment_id),
|
||||
FOREIGN KEY (chat_id, message_id) REFERENCES messages(chat_id, id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (chat_id, attachment_id) REFERENCES attachments(chat_id, id)
|
||||
);
|
||||
@@ -0,0 +1,27 @@
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi==23.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
fastapi==0.115.12
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
psycopg==3.2.6
|
||||
psycopg-binary==3.2.6
|
||||
psycopg-pool==3.2.6
|
||||
pycparser==3.12
|
||||
pydantic==2.11.3
|
||||
pydantic_core==2.33.1
|
||||
starlette==0.46.2
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
uvicorn==0.34.2
|
||||
boto3==1.40.45
|
||||
botocore==1.40.45
|
||||
jmespath==1.0.1
|
||||
Pillow==12.2.0
|
||||
python-dateutil==2.9.0.post0
|
||||
s3transfer==0.14.0
|
||||
six==1.17.0
|
||||
urllib3==2.6.3
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi==0.115.12
|
||||
starlette==0.46.2
|
||||
pydantic==2.11.3
|
||||
uvicorn==0.34.2
|
||||
psycopg[binary]==3.2.6
|
||||
psycopg-pool==3.2.6
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.40.45
|
||||
Pillow==12.2.0
|
||||
@@ -0,0 +1,52 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$root = Split-Path -Parent $PSScriptRoot
|
||||
$utf8 = [Text.UTF8Encoding]::new($false)
|
||||
$secretDir = Join-Path $root 'secrets'
|
||||
[void][IO.Directory]::CreateDirectory($secretDir)
|
||||
|
||||
# Compose bind-mounts these files via Docker Desktop. Keep the host ACL private.
|
||||
$sid = [Security.Principal.WindowsIdentity]::GetCurrent().User
|
||||
if (Test-Path -LiteralPath $secretDir -PathType Container) {
|
||||
$security = [Security.AccessControl.DirectorySecurity]::new()
|
||||
$security.SetOwner($sid)
|
||||
$security.SetAccessRuleProtection($true, $false)
|
||||
$inheritance = [Security.AccessControl.InheritanceFlags]'ContainerInherit, ObjectInherit'
|
||||
foreach ($principal in @($sid, [Security.Principal.SecurityIdentifier]::new('S-1-5-18'))) {
|
||||
$rule = [Security.AccessControl.FileSystemAccessRule]::new(
|
||||
$principal, 'FullControl', $inheritance, 'None', 'Allow'
|
||||
)
|
||||
$security.AddAccessRule($rule)
|
||||
}
|
||||
Set-Acl -LiteralPath $secretDir -AclObject $security
|
||||
}
|
||||
|
||||
function New-HexSecret([string]$name, [int]$length) {
|
||||
$file = Join-Path $secretDir $name
|
||||
if (-not (Test-Path -LiteralPath $file)) {
|
||||
$bytes = New-Object byte[] $length
|
||||
$rng = [Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
try { $rng.GetBytes($bytes) } finally { $rng.Dispose() }
|
||||
$value = [BitConverter]::ToString($bytes).Replace('-', '').ToLowerInvariant()
|
||||
[IO.File]::WriteAllText($file, $value, $utf8)
|
||||
}
|
||||
}
|
||||
New-HexSecret 'postgres_password' 32
|
||||
New-HexSecret 's3_access_key_id' 10
|
||||
New-HexSecret 's3_secret_access_key' 24
|
||||
|
||||
$password = [IO.File]::ReadAllText((Join-Path $secretDir 'postgres_password')).Trim()
|
||||
if ($password -cnotmatch '^[a-f0-9]{64}$') { throw 'Invalid local PostgreSQL secret format.' }
|
||||
$url = Join-Path $secretDir 'database_url'
|
||||
$expected = "postgresql://messenger:${password}@postgres:5432/messenger"
|
||||
if (-not (Test-Path -LiteralPath $url)) {
|
||||
[IO.File]::WriteAllText($url, $expected, $utf8)
|
||||
}
|
||||
if ([IO.File]::ReadAllText($url).Trim() -cne $expected) {
|
||||
throw 'Existing database_url and postgres_password disagree.'
|
||||
}
|
||||
|
||||
$envFile = Join-Path $root '.env'
|
||||
if (-not (Test-Path -LiteralPath $envFile)) {
|
||||
Copy-Item -LiteralPath (Join-Path $root '.env.example') -Destination $envFile
|
||||
}
|
||||
Write-Output 'Lab 4 local configuration ready.'
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# Explicit Compose labels scope the socket to this local test project.
|
||||
case "${LAB4_PROJECT:-}" in
|
||||
mtusi-lab4-grade3|mtusi-lab4-grade3-*) ;;
|
||||
*) echo 'LAB4_PROJECT must identify the Lab 4 test stack' >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
container() {
|
||||
ids=$(docker ps -aq --filter "label=com.docker.compose.project=$LAB4_PROJECT" \
|
||||
--filter "label=com.docker.compose.service=$1" \
|
||||
--filter 'label=com.docker.compose.oneoff=False')
|
||||
set -- $ids
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo 'Expected one Lab 4 service container' >&2
|
||||
return 2
|
||||
fi
|
||||
printf '%s\n' "$1"
|
||||
}
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
. "$(dirname "$0")/lab-containers.sh"
|
||||
postgres=$(container postgres)
|
||||
s3=$(container s3)
|
||||
api1=$(container api1)
|
||||
api2=$(container api2)
|
||||
docker restart --time 4 "$postgres" "$s3"
|
||||
docker restart --time 4 "$api1" "$api2"
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
case "${TARGET_INSTANCE_ID:-}" in
|
||||
api1|api2) ;;
|
||||
*) echo 'TARGET_INSTANCE_ID must be api1 or api2' >&2; exit 2 ;;
|
||||
esac
|
||||
. "$(dirname "$0")/lab-containers.sh"
|
||||
id=$(container "$TARGET_INSTANCE_ID")
|
||||
docker stop --time 4 "$id"
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from io import BytesIO
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from PIL import Image
|
||||
from psycopg_pool import PoolTimeout, TooManyRequests
|
||||
|
||||
from src import s3
|
||||
from src.db import connection
|
||||
from src.errors import ApiError
|
||||
from src.store import public
|
||||
|
||||
|
||||
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
MAX_IMAGE_PIXELS = 40_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
||||
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
|
||||
METADATA = """id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
status, error_code, created_at, updated_at"""
|
||||
|
||||
|
||||
def oversized():
|
||||
return ApiError(413, "payload_too_large", "Превышен допустимый размер изображения")
|
||||
|
||||
|
||||
def invalid_image():
|
||||
return ApiError(415, "unsupported_media_type", "Ожидается корректный PNG или JPEG")
|
||||
|
||||
|
||||
def validate_filename(filename):
|
||||
if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename):
|
||||
raise ApiError(400, "invalid_request", "Недопустимое имя файла")
|
||||
|
||||
|
||||
async def read_upload(request):
|
||||
lengths = request.headers.getlist("content-length")
|
||||
if lengths:
|
||||
if len(lengths) != 1 or not re.fullmatch(r"[0-9]+", lengths[0]):
|
||||
raise ApiError(400, "invalid_request", "Некорректный Content-Length")
|
||||
length = lengths[0].lstrip("0") or "0"
|
||||
if len(length) > len(str(MAX_UPLOAD_BYTES)) or int(length) > MAX_UPLOAD_BYTES:
|
||||
raise oversized()
|
||||
|
||||
body = BytesIO()
|
||||
async for chunk in request.stream():
|
||||
if len(chunk) > MAX_UPLOAD_BYTES - body.tell():
|
||||
raise oversized()
|
||||
body.write(chunk)
|
||||
body.seek(0)
|
||||
return body
|
||||
|
||||
|
||||
def validate_image(body, content_type):
|
||||
try:
|
||||
if content_type == "image/png":
|
||||
# Pillow's verify stops at the IEND chunk before reading its CRC.
|
||||
with body.getbuffer() as data:
|
||||
if data[-12:] != b"\x00\x00\x00\x00IEND\xae\x42\x60\x82":
|
||||
raise invalid_image()
|
||||
with Image.open(body, formats=(FORMATS[content_type],)) as image:
|
||||
if image.width * image.height > MAX_IMAGE_PIXELS:
|
||||
raise oversized()
|
||||
image.verify()
|
||||
with Image.open(body, formats=(FORMATS[content_type],)) as image:
|
||||
# Check each frame of a PNG while bounding the total decode work.
|
||||
for frame in range(getattr(image, "n_frames", 1)):
|
||||
if (frame + 1) * image.width * image.height > MAX_IMAGE_PIXELS:
|
||||
raise oversized()
|
||||
image.seek(frame)
|
||||
image.load()
|
||||
except Image.DecompressionBombError:
|
||||
raise oversized() from None
|
||||
except (OSError, ValueError, SyntaxError, IndexError, EOFError):
|
||||
raise invalid_image() from None
|
||||
finally:
|
||||
body.seek(0)
|
||||
|
||||
|
||||
def upload(chat_id, owner_id, filename, content_type, body):
|
||||
validate_image(body, content_type)
|
||||
with body.getbuffer() as data:
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
size = len(data)
|
||||
attachment_id = uuid4()
|
||||
object_key = f"originals/{uuid4().hex}"
|
||||
s3.put(object_key, body, size, content_type)
|
||||
|
||||
insert_finished = False
|
||||
try:
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
f"""INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING {METADATA}""",
|
||||
(attachment_id, chat_id, owner_id, filename, content_type, size, digest, object_key),
|
||||
).fetchone()
|
||||
insert_finished = True
|
||||
except (psycopg.Error, PoolTimeout, TooManyRequests) as exc:
|
||||
# Compensate before commit or after a definite constraint failure.
|
||||
# A lost commit acknowledgement can mean the row is already durable.
|
||||
if not insert_finished or isinstance(exc, psycopg.IntegrityError):
|
||||
try:
|
||||
s3.delete(object_key)
|
||||
except ApiError:
|
||||
logging.warning("Attachment compensation failed")
|
||||
else:
|
||||
logging.warning("Attachment metadata commit outcome unknown")
|
||||
raise
|
||||
return public({**row, "variants": []})
|
||||
|
||||
|
||||
def get(attachment_id, user_id):
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
f"""SELECT {METADATA}, object_key FROM attachments a
|
||||
WHERE id=%s AND EXISTS (
|
||||
SELECT 1 FROM chat_members m WHERE m.chat_id=a.chat_id AND m.user_id=%s
|
||||
)""",
|
||||
(attachment_id, user_id),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
if row["status"] != "ready":
|
||||
raise ApiError(409, "conflict", "Вложение недоступно")
|
||||
return public({**row, "variants": []})
|
||||
|
||||
|
||||
def original(attachment):
|
||||
data = s3.download(attachment["object_key"], attachment["size_bytes"])
|
||||
if len(data) != attachment["size_bytes"] or hashlib.sha256(data).hexdigest() != attachment["sha256"]:
|
||||
raise s3.unavailable()
|
||||
return data
|
||||
@@ -0,0 +1,16 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def secret(name):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
return Path(path).read_text().strip()
|
||||
raise RuntimeError(f"{name}_FILE is required")
|
||||
|
||||
|
||||
SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
if not 1 <= SESSION_TTL <= 604800:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS must be 1..604800")
|
||||
if os.getenv("SESSION_STORE", "postgres") != "postgres":
|
||||
raise RuntimeError("This implementation uses PostgreSQL sessions")
|
||||
@@ -0,0 +1,40 @@
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from src.config import secret
|
||||
|
||||
pool = ConnectionPool(
|
||||
conninfo=secret("DATABASE_URL"), min_size=1, max_size=8,
|
||||
timeout=2, max_waiting=32, reconnect_timeout=5, open=False,
|
||||
kwargs={"connect_timeout": 2, "row_factory": dict_row,
|
||||
"options": "-c statement_timeout=2000 -c lock_timeout=1500 -c idle_in_transaction_session_timeout=5000 -c timezone=UTC"},
|
||||
)
|
||||
|
||||
|
||||
def migrate():
|
||||
# Serialize migration jobs if two operators start the stack concurrently.
|
||||
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=3) as conn:
|
||||
conn.execute("SET LOCAL lock_timeout = '10s'")
|
||||
conn.execute("SET LOCAL statement_timeout = '15s'")
|
||||
conn.execute("SELECT pg_advisory_xact_lock(202602)")
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())")
|
||||
for path in sorted((Path(__file__).resolve().parents[1] / "migrations").glob("*.sql")):
|
||||
if not conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone():
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (path.name,))
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection():
|
||||
with pool.connection() as conn:
|
||||
# psycopg context commits on success, rolls back on EVERY exception.
|
||||
yield conn
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
migrate()
|
||||
print("PASS: versioned migrations applied (already applied versions skipped)")
|
||||
@@ -0,0 +1,34 @@
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
):
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
def error_response(
|
||||
request: Request,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
) -> JSONResponse:
|
||||
request_id = request.state.request_id
|
||||
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": request_id,
|
||||
}
|
||||
},
|
||||
)
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import anyio
|
||||
import psycopg
|
||||
from fastapi import Depends, FastAPI, Query, Request, Response
|
||||
from starlette.requests import ClientDisconnect
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from psycopg_pool import PoolTimeout, TooManyRequests
|
||||
from starlette.exceptions import HTTPException
|
||||
|
||||
from src import attachments, s3, store
|
||||
from src.config import SESSION_TTL
|
||||
from src.db import connection, pool
|
||||
from src.errors import ApiError, error_response
|
||||
from src.models import Attachment, Chat, ChatPage, CreateChat, CreateMessage, CreateUser, Message, MessagePage, User
|
||||
from src.security import basic_credentials, hasher, token_hash, unauthorized, verify
|
||||
|
||||
|
||||
INSTANCE_ID = os.getenv("INSTANCE_ID") or uuid4().hex
|
||||
IDEMPOTENCY_KEY = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
anyio.to_thread.current_default_thread_limiter().total_tokens = 16
|
||||
app.state.upload_slots = anyio.Semaphore(2)
|
||||
pool.open()
|
||||
try:
|
||||
pool.wait(timeout=10)
|
||||
yield
|
||||
finally:
|
||||
pool.close()
|
||||
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger", version="1.4.0", lifespan=lifespan,
|
||||
docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
def failure(request, status, code, message):
|
||||
response = error_response(request, status, code, message)
|
||||
if status == 401:
|
||||
basic = request.url.path == "/api/v1/auth/sessions"
|
||||
response.headers['WWW-Authenticate'] = ('Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"')
|
||||
return response
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_context(request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
try:
|
||||
if request.method == "POST" and request.url.path == "/api/v1/auth/sessions" and await request.body():
|
||||
raise ApiError(400, "invalid_request", "Вход не принимает тело запроса")
|
||||
response = await call_next(request)
|
||||
except ApiError as exc:
|
||||
response = failure(request, exc.status_code, exc.code, exc.message)
|
||||
except (psycopg.Error, PoolTimeout, TooManyRequests):
|
||||
response = failure(request, 503, "unavailable", "Хранилище временно недоступно")
|
||||
except Exception as exc:
|
||||
logging.error("Unhandled request error: %s", type(exc).__name__)
|
||||
response = failure(request, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
response.headers['X-Request-Id'] = request.state.request_id
|
||||
response.headers['X-Instance-Id'] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error(request, exc):
|
||||
return failure(request, exc.status_code, exc.code, exc.message)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
return failure(request, 400, "invalid_request", "Некорректный запрос")
|
||||
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, exc):
|
||||
return failure(request, exc.status_code, "not_found" if exc.status_code == 404 else "invalid_request", "Недопустимый HTTP-запрос")
|
||||
|
||||
|
||||
def require_user(request: Request):
|
||||
digest = token_hash(request.headers.get('Authorization'))
|
||||
with connection() as conn:
|
||||
user = conn.execute(
|
||||
"""SELECT u.id,u.username,u.display_name,u.created_at
|
||||
FROM sessions s JOIN users u ON u.id=s.user_id
|
||||
WHERE s.token_hash=%s AND s.expires_at > clock_timestamp()""",
|
||||
(digest,),
|
||||
).fetchone()
|
||||
if user is None:
|
||||
unauthorized()
|
||||
request.state.token_hash = digest
|
||||
return User(**store.public(user))
|
||||
|
||||
|
||||
@app.get('/health/live')
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get('/health/ready')
|
||||
def ready():
|
||||
with connection() as conn:
|
||||
conn.execute("SELECT 1").fetchone()
|
||||
s3.ready()
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post('/api/v1/users', status_code=201, response_model=User)
|
||||
def register(data: CreateUser):
|
||||
return store.create_user(data, hasher.hash(data.password))
|
||||
|
||||
|
||||
@app.post('/api/v1/auth/sessions', status_code=201)
|
||||
def login(request: Request):
|
||||
username, password = basic_credentials(request.headers.get('Authorization'))
|
||||
with connection() as conn:
|
||||
row = conn.execute("SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s", (username,)).fetchone()
|
||||
valid = verify(row['password_hash'] if row else None, password)
|
||||
if not row or not valid:
|
||||
unauthorized()
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = hashlib.sha256(token.encode('ascii')).digest()
|
||||
with connection() as conn:
|
||||
result = conn.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at", (digest, row['id'], SESSION_TTL)).fetchone()
|
||||
del row['password_hash']
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": store.public(result['expires_at']), "user": store.public(row)}
|
||||
|
||||
|
||||
@app.delete('/api/v1/auth/sessions/current', status_code=204)
|
||||
def logout(request: Request, user: User = Depends(require_user)):
|
||||
with connection() as conn:
|
||||
conn.execute("DELETE FROM sessions WHERE token_hash=%s", (request.state.token_hash,))
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get('/api/v1/users/me', response_model=User)
|
||||
def me(user: User = Depends(require_user)):
|
||||
return user
|
||||
|
||||
|
||||
@app.post('/api/v1/chats', status_code=201, response_model=Chat)
|
||||
def create_chat(data: CreateChat, user: User = Depends(require_user)):
|
||||
return store.create_chat(data, user.id)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats', response_model=ChatPage)
|
||||
def chats(user: User = Depends(require_user), limit: int = Query(50, ge=1, le=100), cursor: str | None = Query(None, min_length=1, max_length=2048)):
|
||||
return store.list_chats(user.id, limit, cursor)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}', response_model=Chat)
|
||||
def chat(chat_id: UUID, user: User = Depends(require_user)):
|
||||
return store.get_chat(str(chat_id), user.id)
|
||||
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201, response_model=Message)
|
||||
def send(request: Request, chat_id: UUID, data: CreateMessage, user: User = Depends(require_user)):
|
||||
keys = request.headers.getlist("Idempotency-Key")
|
||||
if len(keys) != 1 or not IDEMPOTENCY_KEY.fullmatch(keys[0]):
|
||||
raise ApiError(400, "invalid_request", "Требуется корректный Idempotency-Key")
|
||||
return store.create_message(
|
||||
str(chat_id), user.id, request.method, keys[0], data.text, data.attachment_ids,
|
||||
)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}/messages', response_model=MessagePage)
|
||||
def history(chat_id: UUID, user: User = Depends(require_user), limit: int = Query(50, ge=1, le=100), cursor: str | None = Query(None, min_length=1, max_length=2048)):
|
||||
return store.list_messages(str(chat_id), user.id, limit, cursor)
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201, response_model=Attachment)
|
||||
async def upload_attachment(
|
||||
request: Request,
|
||||
chat_id: UUID,
|
||||
filename: str = Query(..., min_length=1, max_length=255),
|
||||
user: User = Depends(require_user),
|
||||
):
|
||||
await anyio.to_thread.run_sync(store.get_chat, str(chat_id), user.id)
|
||||
attachments.validate_filename(filename)
|
||||
types = request.headers.getlist("content-type")
|
||||
if len(types) != 1 or types[0] not in attachments.FORMATS:
|
||||
raise attachments.invalid_image()
|
||||
async with request.app.state.upload_slots:
|
||||
try:
|
||||
body = await attachments.read_upload(request)
|
||||
except ClientDisconnect:
|
||||
raise ApiError(400, "invalid_request", "Загрузка прервана") from None
|
||||
with body:
|
||||
return await anyio.to_thread.run_sync(
|
||||
attachments.upload, chat_id, user.id, filename, types[0], body,
|
||||
)
|
||||
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}', response_model=Attachment)
|
||||
def attachment_metadata(attachment_id: UUID, user: User = Depends(require_user)):
|
||||
row = attachments.get(attachment_id, user.id)
|
||||
del row["object_key"]
|
||||
return row
|
||||
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}/content')
|
||||
def attachment_content(
|
||||
attachment_id: UUID, user: User = Depends(require_user), variant: str = Query("original"),
|
||||
):
|
||||
row = attachments.get(attachment_id, user.id)
|
||||
if variant != "original":
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
return Response(attachments.original(row), media_type=row["content_type"],
|
||||
headers={"Cache-Control": "private, no-store"})
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
from typing import Literal
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
strict=True,
|
||||
)
|
||||
|
||||
username: str = Field(
|
||||
min_length=3,
|
||||
max_length=32,
|
||||
pattern=r"^[a-z0-9_]{3,32}$",
|
||||
)
|
||||
|
||||
display_name: str = Field(
|
||||
min_length=1,
|
||||
max_length=100,
|
||||
)
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def display_name_must_not_be_blank(cls, value: str) -> str:
|
||||
if not value.strip() or "\x00" in value:
|
||||
raise ValueError("display_name must not be blank")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("password")
|
||||
@classmethod
|
||||
def password_must_not_be_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("password must not be blank")
|
||||
return value
|
||||
|
||||
|
||||
class User(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
strict=True,
|
||||
)
|
||||
|
||||
title: str = Field(
|
||||
min_length=1,
|
||||
max_length=100,
|
||||
)
|
||||
|
||||
member_ids: list[str] = Field(
|
||||
min_length=1,
|
||||
max_length=100,
|
||||
)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def title_must_not_be_blank(cls, value: str) -> str:
|
||||
if not value.strip() or "\x00" in value:
|
||||
raise ValueError("title must not be blank")
|
||||
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def validate_member_ids(cls, value: list[str]) -> list[str]:
|
||||
normalized = []
|
||||
for member_id in value:
|
||||
try:
|
||||
normalized.append(str(UUID(member_id)))
|
||||
except (ValueError, TypeError):
|
||||
raise ValueError("member_ids must contain valid UUIDs") from None
|
||||
|
||||
if len(normalized) != len(set(normalized)):
|
||||
raise ValueError("member_ids must be unique")
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
class Chat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
title: str
|
||||
member_ids: list[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class ChatPage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
items: list[Chat]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
extra="forbid",
|
||||
strict=True,
|
||||
)
|
||||
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def validate_attachment_ids(cls, value: list[str]) -> list[str]:
|
||||
normalized = [str(UUID(attachment_id)) for attachment_id in value]
|
||||
if len(normalized) != len(set(normalized)):
|
||||
raise ValueError("attachment_ids must be unique")
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_content(self):
|
||||
if "\x00" in self.text or (not self.text.strip() and not self.attachment_ids):
|
||||
raise ValueError("Message requires text or attachments")
|
||||
return self
|
||||
|
||||
|
||||
class Message(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: list[str]
|
||||
|
||||
|
||||
class MessagePage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
items: list[Message]
|
||||
next_cursor: str | None
|
||||
|
||||
class Attachment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: Literal["image/png", "image/jpeg"]
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: Literal["ready"]
|
||||
variants: list
|
||||
error_code: None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
@@ -0,0 +1,74 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from src.errors import ApiError
|
||||
|
||||
|
||||
def encode_cursor(
|
||||
resource: str,
|
||||
owner_id: str,
|
||||
created_at: str,
|
||||
object_id: str,
|
||||
) -> str:
|
||||
payload = {
|
||||
"resource": resource,
|
||||
"owner_id": owner_id,
|
||||
"created_at": created_at,
|
||||
"id": object_id,
|
||||
}
|
||||
|
||||
raw = json.dumps(
|
||||
payload,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(
|
||||
cursor: str,
|
||||
expected_resource: str,
|
||||
expected_owner_id: str,
|
||||
) -> tuple[str, str]:
|
||||
try:
|
||||
if not re.fullmatch(r"[A-Za-z0-9_-]+", cursor):
|
||||
raise ValueError
|
||||
padding = "=" * (-len(cursor) % 4)
|
||||
|
||||
raw = base64.b64decode(
|
||||
(cursor + padding).encode("ascii"), altchars=b"-_", validate=True
|
||||
)
|
||||
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
|
||||
if payload["resource"] != expected_resource:
|
||||
raise ValueError
|
||||
|
||||
if payload["owner_id"] != expected_owner_id:
|
||||
raise ValueError
|
||||
|
||||
created_at = payload["created_at"]
|
||||
object_id = payload["id"]
|
||||
|
||||
# The cursor must carry the same UTC timestamp format produced
|
||||
# by the server. Invalid timestamps must return HTTP 400 before a SQL cast.
|
||||
if not isinstance(created_at, str) or not re.fullmatch(
|
||||
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z",
|
||||
created_at,
|
||||
):
|
||||
raise ValueError
|
||||
|
||||
datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
||||
object_id = str(UUID(object_id))
|
||||
|
||||
return created_at, object_id
|
||||
|
||||
except Exception:
|
||||
raise ApiError(
|
||||
status_code=400,
|
||||
code="invalid_request",
|
||||
message="Некорректный cursor",
|
||||
)
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Synchronous access to the single private object bucket."""
|
||||
|
||||
import os
|
||||
from contextlib import closing
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from src.config import secret
|
||||
from src.errors import ApiError
|
||||
|
||||
|
||||
BUCKET = os.environ["S3_BUCKET"]
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.environ["S3_ENDPOINT"],
|
||||
region_name=os.environ["S3_REGION"],
|
||||
aws_access_key_id=secret("S3_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=secret("S3_SECRET_ACCESS_KEY"),
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=0.5,
|
||||
read_timeout=2,
|
||||
retries={"total_max_attempts": 1},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
proxies={},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def unavailable():
|
||||
return ApiError(503, "unavailable", "Хранилище вложений временно недоступно")
|
||||
|
||||
|
||||
def ready():
|
||||
try:
|
||||
client.head_bucket(Bucket=BUCKET)
|
||||
except (ClientError, BotoCoreError):
|
||||
raise unavailable() from None
|
||||
|
||||
|
||||
def put(key, content, size, content_type):
|
||||
try:
|
||||
client.put_object(
|
||||
Bucket=BUCKET, Key=key, Body=content, ContentLength=size,
|
||||
ContentType=content_type,
|
||||
)
|
||||
except (ClientError, BotoCoreError):
|
||||
raise unavailable() from None
|
||||
|
||||
|
||||
def download(key, size):
|
||||
try:
|
||||
result = client.get_object(Bucket=BUCKET, Key=key)
|
||||
with closing(result["Body"]) as body:
|
||||
if result["ContentLength"] != size:
|
||||
raise unavailable()
|
||||
return body.read(size)
|
||||
except (ClientError, BotoCoreError):
|
||||
raise unavailable() from None
|
||||
|
||||
|
||||
def delete(key):
|
||||
try:
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
except (ClientError, BotoCoreError):
|
||||
raise unavailable() from None
|
||||
|
||||
|
||||
def bootstrap():
|
||||
try:
|
||||
client.head_bucket(Bucket=BUCKET)
|
||||
except ClientError as exc:
|
||||
if exc.response["ResponseMetadata"]["HTTPStatusCode"] != 404:
|
||||
raise
|
||||
client.create_bucket(Bucket=BUCKET)
|
||||
# New buckets are private by default. Remove any old public bucket policy.
|
||||
client.delete_bucket_policy(Bucket=BUCKET)
|
||||
client.head_bucket(Bucket=BUCKET)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
bootstrap()
|
||||
except (ClientError, BotoCoreError):
|
||||
raise SystemExit("S3 bootstrap failed") from None
|
||||
print("PASS: private attachment bucket bootstrapped")
|
||||
@@ -0,0 +1,47 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError, InvalidHashError
|
||||
|
||||
from src.errors import ApiError
|
||||
|
||||
# Explicit bounded Argon2id parameters: 19 MiB, 2 passes, one lane.
|
||||
hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
DUMMY_HASH = hasher.hash(secrets.token_urlsafe(32))
|
||||
|
||||
|
||||
def unauthorized():
|
||||
raise ApiError(401, "unauthorized", "Требуется действительная сессия")
|
||||
|
||||
|
||||
def basic_credentials(header):
|
||||
try:
|
||||
scheme, value = header.split(" ", 1)
|
||||
if scheme.lower() != "basic":
|
||||
raise ValueError
|
||||
username, password = base64.b64decode(value, validate=True).decode("utf-8").split(":", 1)
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
|
||||
raise ValueError
|
||||
return username, password
|
||||
except (ValueError, UnicodeError, AttributeError):
|
||||
unauthorized()
|
||||
|
||||
|
||||
def verify(encoded, password):
|
||||
try:
|
||||
return hasher.verify(encoded or DUMMY_HASH, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def token_hash(header):
|
||||
try:
|
||||
scheme, token = header.split(" ", 1)
|
||||
if scheme.lower() != "bearer" or not re.fullmatch(r"[A-Za-z0-9_-]{43}", token):
|
||||
raise ValueError
|
||||
return hashlib.sha256(token.encode("ascii")).digest()
|
||||
except (ValueError, AttributeError):
|
||||
unauthorized()
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from psycopg.errors import ForeignKeyViolation, UniqueViolation
|
||||
|
||||
from src.db import connection
|
||||
from src.errors import ApiError
|
||||
from src.pagination import decode_cursor, encode_cursor
|
||||
|
||||
|
||||
def public(value):
|
||||
if isinstance(value, dict):
|
||||
return {key: public(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [public(item) for item in value]
|
||||
if isinstance(value, UUID):
|
||||
return str(value)
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat().replace("+00:00", "Z")
|
||||
return value
|
||||
|
||||
|
||||
def create_user(data, password_hash):
|
||||
try:
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at",
|
||||
(uuid4(), data.username, data.display_name, password_hash),
|
||||
).fetchone()
|
||||
return public(row)
|
||||
except UniqueViolation:
|
||||
raise ApiError(409, "conflict", "Username уже занят") from None
|
||||
|
||||
|
||||
def create_chat(data, caller):
|
||||
if caller not in data.member_ids:
|
||||
raise ApiError(400, "invalid_request", "Создатель должен входить в member_ids")
|
||||
try:
|
||||
with connection() as conn:
|
||||
chat = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), data.title)).fetchone()
|
||||
# FK errors here MUST roll back the already inserted parent chat.
|
||||
with conn.cursor() as cursor:
|
||||
cursor.executemany("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", [(chat['id'], member) for member in data.member_ids])
|
||||
chat['member_ids'] = data.member_ids
|
||||
return public(chat)
|
||||
except ForeignKeyViolation:
|
||||
raise ApiError(400, "invalid_request", "Указан неизвестный участник") from None
|
||||
|
||||
|
||||
CHAT_SELECT = """SELECT c.id,c.title,c.created_at,
|
||||
ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id) AS member_ids
|
||||
FROM chats c WHERE EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"""
|
||||
|
||||
|
||||
def get_chat(chat_id, user_id):
|
||||
with connection() as conn:
|
||||
row = conn.execute(CHAT_SELECT + " AND c.id=%s", (user_id, chat_id)).fetchone()
|
||||
if not row:
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
return public(row)
|
||||
|
||||
|
||||
def page(rows, limit, resource, owner):
|
||||
items = public(rows[:limit])
|
||||
cursor = None
|
||||
if len(rows) > limit:
|
||||
last = items[-1]
|
||||
cursor = encode_cursor(resource, owner, last['created_at'], last['id'])
|
||||
return {"items": items, "next_cursor": cursor}
|
||||
|
||||
|
||||
def list_chats(user_id, limit, cursor):
|
||||
params = [user_id]
|
||||
sql = CHAT_SELECT
|
||||
if cursor is not None:
|
||||
timestamp, object_id = decode_cursor(cursor, "chats", user_id)
|
||||
sql += " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)"
|
||||
params += [timestamp, object_id]
|
||||
sql += " ORDER BY c.created_at,c.id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
with connection() as conn:
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
return page(rows, limit, "chats", user_id)
|
||||
|
||||
|
||||
MESSAGE_SELECT = """SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,
|
||||
ARRAY(SELECT ma.attachment_id FROM message_attachments ma
|
||||
WHERE ma.message_id=m.id ORDER BY ma.position) AS attachment_ids
|
||||
FROM messages m"""
|
||||
|
||||
|
||||
def check_attachments(conn, chat_id, attachment_ids):
|
||||
if not attachment_ids:
|
||||
return
|
||||
rows = conn.execute(
|
||||
"""SELECT id,status FROM attachments
|
||||
WHERE chat_id=%s AND id=ANY(%s::uuid[]) ORDER BY id FOR SHARE""",
|
||||
(chat_id, attachment_ids),
|
||||
).fetchall()
|
||||
if len(rows) != len(attachment_ids):
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
if any(row["status"] != "ready" for row in rows):
|
||||
raise ApiError(409, "conflict", "Вложение недоступно")
|
||||
|
||||
|
||||
def create_message(chat_id, user_id, method, key, text, attachment_ids):
|
||||
scope = (user_id, method, chat_id, key)
|
||||
message_id = uuid4()
|
||||
with connection() as conn:
|
||||
if not conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s FOR KEY SHARE",
|
||||
(chat_id, user_id),
|
||||
).fetchone():
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
|
||||
# A concurrent ON CONFLICT waits for the first transaction. The deferred
|
||||
# FK allows reserving the proposed message ID before inserting the row.
|
||||
claimed = conn.execute(
|
||||
"""INSERT INTO message_idempotency (user_id,method,chat_id,key,message_id,expires_at)
|
||||
VALUES (%s,%s,%s,%s,%s,clock_timestamp() + interval '24 hours')
|
||||
ON CONFLICT (user_id,method,chat_id,key) DO UPDATE
|
||||
SET message_id=EXCLUDED.message_id, expires_at=clock_timestamp() + interval '24 hours'
|
||||
WHERE message_idempotency.expires_at <= clock_timestamp()
|
||||
RETURNING message_id""",
|
||||
(*scope, message_id),
|
||||
).fetchone()
|
||||
|
||||
if claimed:
|
||||
check_attachments(conn, chat_id, attachment_ids)
|
||||
row = conn.execute(
|
||||
"""INSERT INTO messages (id,chat_id,sender_id,text)
|
||||
VALUES (%s,%s,%s,%s)
|
||||
RETURNING id,chat_id,sender_id,text,created_at""",
|
||||
(message_id, chat_id, user_id, text),
|
||||
).fetchone()
|
||||
if attachment_ids:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.executemany(
|
||||
"""INSERT INTO message_attachments
|
||||
(chat_id,message_id,attachment_id,position) VALUES (%s,%s,%s,%s)""",
|
||||
[
|
||||
(chat_id, message_id, attachment_id, position)
|
||||
for position, attachment_id in enumerate(attachment_ids, 1)
|
||||
],
|
||||
)
|
||||
row["attachment_ids"] = attachment_ids
|
||||
else:
|
||||
# READ COMMITTED gives this statement a new snapshot. The UPSERT
|
||||
# still holds the key row locked, including when its WHERE was false.
|
||||
row = conn.execute(
|
||||
MESSAGE_SELECT + """ JOIN message_idempotency i ON i.message_id=m.id
|
||||
WHERE (i.user_id,i.method,i.chat_id,i.key)=(%s,%s,%s,%s)""",
|
||||
scope,
|
||||
).fetchone()
|
||||
if row["text"] != text or [str(value) for value in row["attachment_ids"]] != attachment_ids:
|
||||
raise ApiError(409, "conflict", "Ключ уже использован с другим сообщением")
|
||||
return public(row)
|
||||
|
||||
|
||||
def list_messages(chat_id, user_id, limit, cursor):
|
||||
with connection() as conn:
|
||||
# Access precedes cursor decoding, including malformed foreign cursors.
|
||||
if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone():
|
||||
raise ApiError(404, "not_found", "Объект не найден")
|
||||
params = [chat_id]
|
||||
sql = MESSAGE_SELECT + " WHERE m.chat_id=%s"
|
||||
if cursor is not None:
|
||||
timestamp, object_id = decode_cursor(cursor, "messages", chat_id)
|
||||
sql += " AND (created_at,id) > (%s::timestamptz,%s::uuid)"
|
||||
params += [timestamp, object_id]
|
||||
sql += " ORDER BY created_at,id LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
return page(rows, limit, "messages", chat_id)
|
||||
@@ -0,0 +1,327 @@
|
||||
"""Black-box HTTP and real Docker fault checks for the local Lab 4 project."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
from http.client import HTTPConnection
|
||||
from io import BytesIO
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from threading import Barrier
|
||||
import time
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check, run, shape
|
||||
|
||||
REPLICAS = {"api1", "api2"}
|
||||
SERVICES = ("postgres", "s3", "api1", "api2")
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(
|
||||
("docker", *args), text=True, timeout=30, stderr=subprocess.DEVNULL
|
||||
).strip()
|
||||
|
||||
|
||||
def container(service):
|
||||
project = os.environ["LAB4_PROJECT"]
|
||||
check(re.fullmatch(r"mtusi-lab4-grade3(?:-[a-z0-9-]+)?", project), "unexpected project")
|
||||
ids = docker("ps", "-aq", "--filter", f"label=com.docker.compose.project={project}",
|
||||
"--filter", f"label=com.docker.compose.service={service}",
|
||||
"--filter", "label=com.docker.compose.oneoff=False").splitlines()
|
||||
check(len(ids) == 1, "expected exactly one service container")
|
||||
return ids[0]
|
||||
|
||||
|
||||
def state(service):
|
||||
row = docker("inspect", "--format",
|
||||
"{{.Id}} {{.State.Running}} {{.State.StartedAt}} {{.State.FinishedAt}}",
|
||||
container(service)).split()
|
||||
return dict(zip(("id", "running", "started", "finished"), row))
|
||||
|
||||
|
||||
def hook(name, **env):
|
||||
subprocess.run(
|
||||
[str(ROOT / "scripts" / name)], env={**os.environ, **env}, timeout=30,
|
||||
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
def healthy(client, wanted, timeout=20):
|
||||
deadline = time.monotonic() + timeout
|
||||
instances = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
response = client.call("GET", "/health/ready", schema="Health")
|
||||
instances.add(response["headers"]["X-Instance-Id"])
|
||||
if instances == wanted:
|
||||
return instances
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise Failure("expected instances not observed in readiness")
|
||||
|
||||
|
||||
def image(fmt, size=(20, 12), mode="RGB"):
|
||||
with Image.new(mode, size) as fixture, BytesIO() as stream:
|
||||
fixture.save(stream, fmt)
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def upload(client, actor, chat, data, mime="image/png", filename="фото.png", status=201):
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename={quote(filename, safe='')}"
|
||||
response = client.call("POST", path, status, actor=actor, raw=data,
|
||||
headers={"Content-Type": mime}, schema="Attachment" if status == 201 else None)
|
||||
if status == 201:
|
||||
metadata = response["json"]
|
||||
check(metadata["sha256"] == hashlib.sha256(data).hexdigest(), "metadata checksum")
|
||||
check(metadata["size_bytes"] == len(data), "metadata size")
|
||||
check(metadata["filename"] == filename, "metadata filename")
|
||||
check(metadata["chat_id"] == chat["id"] and metadata["owner_id"] == actor["id"], "metadata ownership")
|
||||
check(metadata["variants"] == [] and metadata["error_code"] is None, "ready metadata")
|
||||
check(metadata["content_type"] == mime, "metadata MIME")
|
||||
return response
|
||||
|
||||
|
||||
def download(client, actor, metadata, data):
|
||||
path = f"/api/v1/attachments/{metadata['id']}"
|
||||
current = client.call("GET", path, actor=actor, schema="Attachment")
|
||||
check(current["json"] == metadata, "metadata changed")
|
||||
response = client.call("GET", path + "/content", actor=actor)
|
||||
check(response["bytes"] == data, "original changed")
|
||||
check(response["headers"].get_content_type() == metadata["content_type"], "download MIME")
|
||||
check(hashlib.sha256(response["bytes"]).hexdigest() == metadata["sha256"], "download checksum")
|
||||
return response
|
||||
|
||||
|
||||
def chunked_limit(actor, chat):
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=chunked.png"
|
||||
chunk = b"\x00" * (256 * 1024)
|
||||
connection = HTTPConnection("lb", 8080, timeout=6)
|
||||
try:
|
||||
connection.putrequest("POST", path)
|
||||
connection.putheader("Authorization", "Bearer " + actor["token"])
|
||||
connection.putheader("Content-Type", "image/png")
|
||||
connection.putheader("Transfer-Encoding", "chunked")
|
||||
connection.endheaders()
|
||||
for block in [chunk] * 40 + [b"\x00"]:
|
||||
connection.send(f"{len(block):X}\r\n".encode() + block + b"\r\n")
|
||||
connection.send(b"0\r\n\r\n")
|
||||
response = connection.getresponse()
|
||||
check(response.status == 413, "chunked upload exceeded limit")
|
||||
shape(json.loads(response.read()), "Error")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def negatives(client, alice, eve, chat, metadata, png):
|
||||
path = f"/api/v1/attachments/{metadata['id']}"
|
||||
for suffix in ("", "/content"):
|
||||
client.call("GET", path + suffix, 404, actor=eve)
|
||||
client.call("GET", path + suffix, 401)
|
||||
client.call("GET", path + "/content?variant=thumbnail", 404, actor=alice)
|
||||
upload(client, eve, chat, png, status=404)
|
||||
upload(client, None, chat, png, status=401)
|
||||
for filename in ("", "a" * 256, "../photo.png", "sub\\photo.png", "nul\0.png", "ctl\x85.png"):
|
||||
upload(client, alice, chat, png, filename=filename, status=400)
|
||||
upload(client, alice, chat, b"not an image", status=415)
|
||||
upload(client, alice, chat, png[:32], status=415)
|
||||
upload(client, alice, chat, png[:-3], status=415)
|
||||
upload(client, alice, chat, image("JPEG")[:-10], mime="image/jpeg", status=415)
|
||||
upload(client, alice, chat, png, mime="image/jpeg", status=415)
|
||||
upload(client, alice, chat, png, mime="multipart/form-data", status=415)
|
||||
upload(client, alice, chat, b"\x00" * (10 * 1024 * 1024 + 1), status=413)
|
||||
chunked_limit(alice, chat)
|
||||
upload(client, alice, chat, image("PNG", (8001, 5000), "L"), status=413)
|
||||
print("PASS ACL metadata/content/upload: outsider=404 missing Bearer=401; filenames=400; "
|
||||
"damaged/false MIME=415; 10 MiB+1 with/without Content-Length=413; 40,005,000 pixels=413")
|
||||
|
||||
|
||||
def metadata_sql(statement):
|
||||
return docker("exec", container("postgres"), "psql", "-XAtq", "-v", "ON_ERROR_STOP=1",
|
||||
"-U", "messenger", "-d", "messenger", "-c", statement)
|
||||
|
||||
|
||||
def private_bucket(metadata):
|
||||
attachment_id = UUID(metadata["id"])
|
||||
key = metadata_sql(f"SELECT object_key FROM attachments WHERE id='{attachment_id}'")
|
||||
check(re.fullmatch(r"originals/[0-9a-f]{32}", key), "unexpected object key")
|
||||
check(not docker("port", container("s3")), "S3 is published")
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
try:
|
||||
with urlopen(f"http://s3:9000/{quote(bucket)}/{quote(key)}", timeout=4):
|
||||
raise Failure("anonymous direct S3 GET succeeded")
|
||||
except HTTPError as exc:
|
||||
with exc:
|
||||
check(exc.code == 403, "anonymous direct object GET must be forbidden")
|
||||
print("PASS anonymous internal GET of known existing object: 403; Docker publishes no S3 ports")
|
||||
|
||||
|
||||
def idempotency(client, alice, bob, chat, first, second):
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
other_chat = client.call("POST", "/api/v1/chats", 201, actor=alice,
|
||||
body={"title": "Other chat", "member_ids": [alice["id"]]}, schema="Chat")["json"]
|
||||
foreign = upload(client, alice, other_chat, image("PNG"))["json"]["id"]
|
||||
for ids, code in (([foreign], 404), ([str(uuid4())], 404), ([first, first], 400),
|
||||
([str(uuid4()) for _ in range(11)], 400)):
|
||||
client.call("POST", path, code, actor=bob, body={"text": "", "attachment_ids": ids},
|
||||
headers={"Idempotency-Key": uuid4().hex})
|
||||
client.call("POST", path, 400, actor=bob, body={"text": " \t"},
|
||||
headers={"Idempotency-Key": uuid4().hex})
|
||||
try:
|
||||
metadata_sql(f"UPDATE attachments SET status='failed',error_code='test' WHERE id='{UUID(first)}'")
|
||||
client.call("POST", path, 409, actor=bob, body={"text": "", "attachment_ids": [first]},
|
||||
headers={"Idempotency-Key": uuid4().hex})
|
||||
finally:
|
||||
metadata_sql(f"UPDATE attachments SET status='ready',error_code=NULL WHERE id='{UUID(first)}'")
|
||||
|
||||
headers = {"Idempotency-Key": uuid4().hex}
|
||||
barrier = Barrier(2)
|
||||
|
||||
def send(replica, body):
|
||||
endpoint = Client(f"http://{replica}:8080", timeout=5)
|
||||
barrier.wait(timeout=5)
|
||||
return endpoint.call("POST", path, 201, actor=bob, body=body,
|
||||
headers=headers, schema="Message")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
left = executor.submit(send, "api1", {"text": " \t", "attachment_ids": [first, second]})
|
||||
right = executor.submit(send, "api2", {"attachment_ids": [first, second], "text": " \t"})
|
||||
results = [left.result(), right.result()]
|
||||
check({r["headers"]["X-Instance-Id"] for r in results} == REPLICAS, "not cross-replica")
|
||||
attached = results[0]["json"]
|
||||
check(attached == results[1]["json"], "concurrent replay differed")
|
||||
check(attached["attachment_ids"] == [first, second] and attached["sender_id"] == bob["id"],
|
||||
"attachment order or sender incorrect")
|
||||
client.call("POST", path, 409, actor=bob, body={"text": " \t", "attachment_ids": [second]},
|
||||
headers=headers)
|
||||
|
||||
key = uuid4().hex
|
||||
old = client.message(bob, chat, "ordinary text", key=key)["json"]
|
||||
check(old == client.message(bob, chat, "ordinary text", key=key, attachments=[])["json"],
|
||||
"omitted and empty attachments differed")
|
||||
page1 = client.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"]
|
||||
check(page1["items"] == [attached] and page1["next_cursor"], "first page lost attachments")
|
||||
page2 = client.call("GET", path + "?limit=1&cursor=" + quote(page1["next_cursor"]),
|
||||
actor=alice, schema="MessagePage")["json"]
|
||||
check(page2 == {"items": [old], "next_cursor": None}, "second page duplicated or lost message")
|
||||
print("PASS concurrent attached POST: api1/api2=201 same message; one history row; changed IDs=409; "
|
||||
"omitted=[]; both cursor pages; foreign/unknown=404, failed=409, invalid lists=400")
|
||||
|
||||
|
||||
def attachment_failover(client, alice, bob, chat, png):
|
||||
for target in ("api1", "api2"):
|
||||
healthy(client, REPLICAS)
|
||||
for _ in range(6):
|
||||
response = upload(client, alice, chat, png)
|
||||
if response["headers"]["X-Instance-Id"] == target:
|
||||
break
|
||||
else:
|
||||
raise Failure("upload did not reach target through HAProxy")
|
||||
other = (REPLICAS - {target}).pop()
|
||||
before = state(target)
|
||||
survivor = state(other)
|
||||
check(before["running"] == survivor["running"] == "true", "APIs not running before stop")
|
||||
started = time.monotonic()
|
||||
try:
|
||||
hook("stop-one.sh", TARGET_INSTANCE_ID=target)
|
||||
stopped = state(target)
|
||||
check(stopped["running"] == "false" and stopped["id"] == before["id"],
|
||||
"the upload-serving container was not stopped")
|
||||
check(state(other) == survivor, "surviving container changed")
|
||||
metadata = response["json"]
|
||||
path = f"/api/v1/attachments/{metadata['id']}/content"
|
||||
while True:
|
||||
try:
|
||||
after = client.call("GET", path, actor=bob)
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic() - started < 10, "download recovery exceeded ten seconds")
|
||||
time.sleep(0.1)
|
||||
elapsed = time.monotonic() - started
|
||||
check(elapsed <= 10 and after["headers"]["X-Instance-Id"] == other, "wrong failover")
|
||||
check(after["bytes"] == png and hashlib.sha256(after["bytes"]).hexdigest() == metadata["sha256"],
|
||||
"failover changed original")
|
||||
print(f"PASS failover upload={target} 201 stopped={stopped['id'][:12]} "
|
||||
f"FinishedAt={stopped['finished']} download={other} 200 "
|
||||
f"seconds={elapsed:.3f} sha256={metadata['sha256']}")
|
||||
finally:
|
||||
docker("start", before["id"])
|
||||
healthy(client, REPLICAS)
|
||||
|
||||
|
||||
def outage_and_restart(client, alice, bob, chat, metadata, png):
|
||||
store = container("s3")
|
||||
direct = Client("http://api1:8080", timeout=4)
|
||||
try:
|
||||
docker("stop", "--time", "4", store)
|
||||
check(state("s3")["running"] == "false", "S3 did not stop")
|
||||
upload(direct, alice, chat, png, status=503)
|
||||
direct.call("GET", f"/api/v1/attachments/{metadata['id']}/content", 503, actor=bob)
|
||||
direct.call("GET", "/health/ready", 503)
|
||||
direct.call("GET", "/health/live", schema="Health")
|
||||
for method, path in (
|
||||
("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=outage.png"),
|
||||
("GET", f"/api/v1/attachments/{metadata['id']}/content"),
|
||||
):
|
||||
headers = {"Authorization": "Bearer " + (alice if method == "POST" else bob)["token"]}
|
||||
if method == "POST":
|
||||
headers["Content-Type"] = "image/png"
|
||||
request = Request(client.base + path, headers=headers, method=method,
|
||||
data=png if method == "POST" else None)
|
||||
try:
|
||||
with urlopen(request, timeout=4):
|
||||
raise Failure("HAProxy unexpectedly accepted the outage request")
|
||||
except HTTPError as exc:
|
||||
with exc:
|
||||
check(exc.code == 503, "HAProxy did not return 503 during S3 outage")
|
||||
print(f"PASS S3 stopped={store[:12]}: LB upload/download=503; API JSON upload/download/ready=503 live=200")
|
||||
finally:
|
||||
docker("start", store)
|
||||
healthy(client, REPLICAS)
|
||||
download(client, bob, metadata, png)
|
||||
|
||||
before = {name: state(name) for name in SERVICES}
|
||||
hook("restart.sh")
|
||||
healthy(client, REPLICAS)
|
||||
for name in SERVICES:
|
||||
after = state(name)
|
||||
check(after["id"] == before[name]["id"] and after["running"] == "true", "container replaced")
|
||||
check(after["started"] > before[name]["started"], "service did not restart")
|
||||
print(f"restart {name} {after['id'][:12]} StartedAt={before[name]['started']} -> {after['started']}")
|
||||
download(client, bob, metadata, png)
|
||||
print(f"PASS original Bearer metadata/content=200 after restart sha256={metadata['sha256']}")
|
||||
|
||||
|
||||
def main():
|
||||
client = Client(timeout=3)
|
||||
check(healthy(client, REPLICAS) == REPLICAS, "expected both APIs")
|
||||
print("PASS HAProxy readiness: api1=200 api2=200")
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
png = image("PNG")
|
||||
jpeg = image("JPEG")
|
||||
first = upload(client, alice, chat, png)["json"]
|
||||
second = upload(client, alice, chat, jpeg, "image/jpeg", "photo.jpg")["json"]
|
||||
for metadata, data in ((first, png), (second, jpeg)):
|
||||
download(client, bob, metadata, data)
|
||||
print(f"PASS {metadata['content_type']} upload=201 ready download=200 bytes={len(data)} "
|
||||
f"sha256={metadata['sha256']}")
|
||||
negatives(client, alice, eve, chat, first, png)
|
||||
private_bucket(first)
|
||||
idempotency(client, alice, bob, chat, first["id"], second["id"])
|
||||
attachment_failover(client, alice, bob, chat, png)
|
||||
outage_and_restart(client, alice, bob, chat, first, png)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user