ЛР 4 — БВТ2302 — Бадрадин Али — на 4 #138
+7
-20
@@ -1,20 +1,7 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# Values substituted by compose.yaml. Credentials stay in local files under secrets/.
|
||||
APP_PORT=8080
|
||||
SESSION_TTL_SECONDS=3600
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
ORPHAN_GRACE_SECONDS=86400
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
FROM python:3.12.6-slim-bookworm
|
||||
|
||||
ARG BUILD_REVISION=local
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONUTF8=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_BUILD_REVISION=${BUILD_REVISION}
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir --requirement requirements.txt
|
||||
|
||||
COPY src/ ./src/
|
||||
COPY migrations/ ./migrations/
|
||||
|
||||
RUN useradd --create-home --uid 10001 appuser
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=3s --timeout=2s --start-period=10s --retries=3 \
|
||||
CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)"
|
||||
|
||||
CMD ["uvicorn", "src.app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1", "--timeout-graceful-shutdown", "5"]
|
||||
@@ -1,30 +1,45 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
|
||||
validate:
|
||||
$(PYTHON) scripts/validate_openapi.py
|
||||
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover test-idempotency test-grade4 test-s3-outage memory-uploads reconcile-s3
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
|
||||
validate:
|
||||
$(PYTHON) scripts/validate_openapi.py
|
||||
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-idempotency:
|
||||
PYTHONPATH=tests $(PYTHON) student-tests/idempotency_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-grade4:
|
||||
PYTHONPATH=tests $(PYTHON) student-tests/grade4_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-s3-outage:
|
||||
PYTHONPATH=tests $(PYTHON) student-tests/s3_outage.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
memory-uploads:
|
||||
PYTHONPATH=tests $(PYTHON) student-tests/upload_memory.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
reconcile-s3:
|
||||
docker compose --env-file .env --profile maintenance run --rm s3-maintenance
|
||||
|
||||
@@ -1,37 +1,105 @@
|
||||
# Отчёт — лабораторная 4
|
||||
# Отчёт по лабораторной работе 4 — вложения в S3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор / группа: Бадрадин Али / БВТ2302
|
||||
- Целевая оценка: 4
|
||||
- Трек 5: не заявлен
|
||||
- Репозиторий: `Бадрадин Али/mtusi2026lab4`
|
||||
- Commit / PR: заполнить после commit и создания PR
|
||||
- Текущее состояние: реализация и перечисленные ниже проверки выполнены локально в Docker Desktop/WSL; результаты подтверждены выводом команд и файлами в `evidence/`. Commit и PR ещё нужно оформить.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Как запустить из чистого клона
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Git, Docker Desktop с запущенным Docker Engine, Python 3.12 и PowerShell. Команды приложения запускаются из корня репозитория.
|
||||
|
||||
```powershell
|
||||
git clone https://gitea.privcloud.tech/Badradine/mtusi2026lab4.git
|
||||
Set-Location mtusi2026lab4
|
||||
Copy-Item .env.example .env
|
||||
.\scripts\create-secrets.ps1
|
||||
python -m venv .venv
|
||||
.\.venv\Scripts\Activate.ps1
|
||||
python -m pip install -r requirements-dev.txt
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --scale api=2
|
||||
docker compose --env-file .env ps
|
||||
```
|
||||
|
||||
Дождаться, пока обе API-реплики, PostgreSQL, Redis, RustFS и балансировщик будут готовы. Затем выполнить:
|
||||
|
||||
```powershell
|
||||
python scripts/check_contract.py
|
||||
python scripts/validate_openapi.py
|
||||
python tests/smoke.py --base-url http://localhost:8080
|
||||
python student-tests/idempotency_concurrency.py --base-url http://localhost:8080
|
||||
python student-tests/grade4_attachments.py --base-url http://localhost:8080
|
||||
python student-tests/s3_outage.py --base-url http://localhost:8080
|
||||
python student-tests/upload_memory.py --base-url http://localhost:8080 --concurrency 10
|
||||
docker compose --env-file .env --profile maintenance run --rm s3-maintenance
|
||||
```
|
||||
|
||||
`upload_memory.py` создаёт сводку в `evidence/upload-memory.json`. Для проверки persistence/failover требуется WSL или Git Bash, так как hooks — shell-скрипты. В WSL из корня клона:
|
||||
|
||||
```sh
|
||||
chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080
|
||||
bash scripts/recover-api.sh
|
||||
```
|
||||
|
||||
Фактические результаты текущего запуска записаны в таблице ниже. Сводка persistence/failover основана на выводе WSL, предоставленном автором; остальные JSON-файлы находятся в `evidence/`. Не сохранять пароли, bearer tokens или содержимое файлов-секретов.
|
||||
|
||||
Для обычной остановки без удаления данных: `docker compose --env-file .env stop`. Команда `docker compose --env-file .env down -v` намеренно удаляет учебные volumes PostgreSQL, Redis и RustFS вместе с их данными; применять только если тестовый стенд нужно полностью сбросить.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Клиент → Nginx → API × 2 → PostgreSQL (пользователи, чаты, сообщения, метаданные)
|
||||
→ Redis (сессии)
|
||||
→ приватный RustFS/S3 bucket (байты изображений)
|
||||
```
|
||||
|
||||
Сохранены возможности Lab 3: PostgreSQL, Redis, bearer-сессии, Argon2id, миграции, две API-реплики, Nginx и идемпотентность POST сообщений. Добавлены таблицы `attachments` и `message_attachments`, синхронные upload/download PNG/JPEG и связь вложений с чатом и сообщением. Составной внешний ключ в БД не позволяет связать сообщение с вложением из другого чата.
|
||||
|
||||
Байты записываются в S3-совместимый RustFS, а PostgreSQL хранит ключ, MIME, имя для отображения, размер, SHA-256 и статус. Ключ объекта создаёт сервер; имя файла никогда не становится путём или S3-ключом. API возвращает `201 ready` только после записи объекта и commit метаданных. При ошибке после S3 PUT выполняется удаление объекта как компенсация. Если процесс аварийно остановится между двумя операциями и компенсация не выполнится, `s3-maintenance` обнаруживает старые объекты без строки БД. По умолчанию это только dry-run; автоматическое удаление возможно только с `--apply` и grace period не меньше одного часа. Миграция `004_attachment_schema_compat` без удаления данных приводит ранее созданную схему `003_attachments` к текущим полям metadata и составной связи с чатом.
|
||||
|
||||
Upload читает тело потоково в spooled temporary file: до 1 MiB хранится в памяти, остальное уходит во временный файл контейнера, размер ограничен 10 MiB. Семафор ограничивает параллельные декодирования каждого API-процесса одним запросом; временный `/tmp` имеет лимит 64 MiB. До полной декомпрессии проверяются PNG/JPEG signature/формат, совпадение MIME, размер изображения и предел 40 млн пикселей. S3 timeout ограничен, SDK настроен на одну попытку.
|
||||
|
||||
Bootstrap создаёт bucket, если его ещё нет, задаёт отдельную service-account key для API с правами только `GetBucketLocation`, `GetObject`, `PutObject` и `DeleteObject` в этом bucket, проверяет приватность анонимным чтением и отсутствие доступа к посторонним bucket. RustFS может вернуть успешный `ListBuckets` с отфильтрованным перечнем разрешённых bucket; bootstrap принимает такой ответ, только если в нём нет bucket вне хранилища вложений. Root-секреты доступны только bootstrap и вручную запускаемому maintenance-сервису; API получает отдельные ограниченные ключи. Перед запуском RustFS одноразовый `s3-init` выставляет владельца тома `s3_data` в UID/GID `10001:10001`, которые используются непривилегированным процессом RustFS. Порты S3 и Console не опубликованы на хосте, встроенная административная Console выключена. Образ RustFS зафиксирован на `rustfs/rustfs:1.0.5`; данные лежат в named volume `s3_data`. Это локальная одноузловая учебная конфигурация, не отказоустойчивая production-система.
|
||||
|
||||
### Соответствие конфигурации
|
||||
|
||||
`.env.example` содержит только используемые Compose-переменные: `APP_PORT`, `SESSION_TTL_SECONDS`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` и `ORPHAN_GRACE_SECONDS`. Реальные URI БД/Redis и S3 credentials монтируются из локальных файлов в `secrets/`. Файл `.env` и каталог `secrets/` исключены из Git. `scripts/create-secrets.ps1` создаёт отсутствующие пары, не печатает значения и не меняет уже созданные полные пары.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
Таблица отражает фактические результаты, полученные автором на локальном стенде. `check_contract.py` и `validate_openapi.py` проверяют структуру/схему, но сами по себе не запускают приложение.
|
||||
|
||||
| Критерий | Команда | Наблюдаемый результат / доказательство | Состояние |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Структура и синтаксис Python | `python scripts/check_contract.py` | `PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.` | PASS |
|
||||
| Полная OpenAPI-валидация | `python scripts/validate_openapi.py` | `PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.` | PASS |
|
||||
| Публичный контракт, доступы и checksum оригинала | `python tests/smoke.py --base-url http://localhost:8080` | Smoke-сценарий Lab 4 завершился `PASS` после применения миграции совместимости 004 | PASS |
|
||||
| Сохранение Lab 3 и идемпотентность на репликах | `python student-tests/idempotency_concurrency.py --base-url http://localhost:8080` | 24 параллельных запроса, 12 workers, обнаружены 2 реплики, одна сохранённая копия | PASS |
|
||||
| Слишком большой файл, повреждённый PNG, MIME spoof, опасное имя | `python student-tests/grade4_attachments.py --base-url http://localhost:8080` | Коды 413, 415, 415 и 400; сводка `evidence/grade4-attachments.json` | PASS |
|
||||
| Чужой и неизвестный attachment ID | Та же команда grade-4 негативных проверок | Для обоих случаев возвращён 404; сводка `evidence/grade4-attachments.json` | PASS |
|
||||
| Недоступность S3 и восстановление | `python student-tests/s3_outage.py --base-url http://localhost:8080` | Liveness 200; readiness и upload во время сбоя 503; после восстановления readiness 200. `evidence/s3-outage.json` | PASS |
|
||||
| Ограниченная потоковая память при ≥10 upload | `python student-tests/upload_memory.py --base-url http://localhost:8080 --concurrency 10` | 10/10 upload успешны, обе реплики наблюдались; RAM до 134.8/143.0 MiB, пик 150.2/151.0 MiB. `evidence/upload-memory.json` | PASS |
|
||||
| Идемпотентный bootstrap, приватный bucket, ограниченные права | `docker compose --env-file .env up -d --build --scale api=2` | `s3-bootstrap` завершился успешно: приватный bucket и ограниченные API credentials готовы; API и LB здоровы | PASS |
|
||||
| Persistence всех хранилищ и прежней сессии/файла | `make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080` | Существующие подтверждённые данные и сессия сохранились после restart. См. `evidence/resilience-results.md` | PASS |
|
||||
| Отказ одной API-реплики и восстановление через LB | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080` | 2 реплики наблюдались до отказа; 84 успешные записи, 0 неудачных попыток, первое восстановление за 1.578 с. Затем `recover-api.sh`: 2 реплики здоровы через LB. См. `evidence/resilience-results.md` | PASS |
|
||||
| Dry-run поиска S3 orphan | `docker compose --env-file .env --profile maintenance run --rm s3-maintenance` | `mode=dry-run`, scanned=13, stale=0, deleted=0 | PASS |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Ключи, секреты, содержимое secret-файлов и bearer tokens не включать в отчёт. Не отмечать checklist PR до подтверждения пунктов выводом.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
В этой версии не заявлен трек 5. Перед commit/PR нужно проверить `docker compose ps`, просмотреть `git diff`/`git status` и убедиться, что секреты не попали в Git. Вывод resilience сообщает `Hook scope needs operator evidence`; результаты подтверждают поведение сценариев теста, а не внешнее подтверждение администраторских действий. RustFS в Compose — одиночный локальный узел с одним volume, поэтому его отказоустойчивость и сохранность диска не эквивалентны кластерному object storage или backup.
|
||||
|
||||
## Источники и инструменты
|
||||
|
||||
- RustFS закреплённый release `1.0.5`: https://github.com/RustFS/RustFS/releases/tag/1.0.5
|
||||
- RustFS service accounts и ограничивающие policy: https://docs.rustfs.com/en/security-compliance/iam/sts и https://docs.rustfs.com/en/security-compliance/iam/policies
|
||||
- RustFS readiness endpoint: https://docs.rustfs.com/en/operations/status-check
|
||||
- boto3 `1.44.1`: https://pypi.org/project/boto3/1.44.1/
|
||||
- Pillow `13.0.0`: https://pypi.org/project/pillow/13.0.0/
|
||||
- Для реализации и ревью использован Codex; проверки запускал автор на локальном стенде. Приведённые результаты относятся к этому запуску и не заменяют повторную проверку при изменении кода или окружения.
|
||||
|
||||
+292
@@ -0,0 +1,292 @@
|
||||
name: mtusi-messenger-lab4
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:16.4-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
networks:
|
||||
- backend
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 1G
|
||||
|
||||
redis:
|
||||
image: redis:7.4.1-alpine
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- exec redis-server --dir /data --appendonly yes --appendfsync everysec --save 60 1 --requirepass "$$(cat /run/secrets/redis_password)"
|
||||
secrets:
|
||||
- redis_password
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
networks:
|
||||
- backend
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- 'REDISCLI_AUTH="$$(cat /run/secrets/redis_password)" redis-cli ping | grep -q PONG'
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 256M
|
||||
|
||||
s3-init:
|
||||
image: alpine:3.24.1
|
||||
user: "0:0"
|
||||
command: ["sh", "-ec", "chown -R 10001:10001 /data && chmod 0750 /data"]
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
restart: "no"
|
||||
|
||||
s3:
|
||||
image: rustfs/rustfs:1.0.5
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
RUSTFS_VOLUMES: /data
|
||||
RUSTFS_ADDRESS: 0.0.0.0:9000
|
||||
RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001
|
||||
RUSTFS_CONSOLE_ENABLE: "false"
|
||||
RUSTFS_OBS_LOGGER_LEVEL: warn
|
||||
RUSTFS_CHECK_UPDATE: "false"
|
||||
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key_id
|
||||
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_secret_access_key
|
||||
secrets:
|
||||
- s3_root_access_key_id
|
||||
- s3_root_secret_access_key
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready >/dev/null"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 1G
|
||||
|
||||
migrate:
|
||||
image: mtusi-messenger-lab4-api:local
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-local}
|
||||
command: ["python", "-m", "src.app.migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 256M
|
||||
|
||||
s3-bootstrap:
|
||||
image: mtusi-messenger-lab4-api:local
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-local}
|
||||
command: ["python", "-m", "src.app.s3_bootstrap"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ROOT_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key_id
|
||||
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets:
|
||||
- s3_root_access_key_id
|
||||
- s3_root_secret_access_key
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 256M
|
||||
|
||||
api:
|
||||
image: mtusi-messenger-lab4-api:local
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-local}
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 8s
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
networks:
|
||||
- backend
|
||||
- edge
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2)"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
tmpfs:
|
||||
- /tmp:size=64m,noexec,nosuid
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512M
|
||||
|
||||
lb:
|
||||
image: nginx:1.28.0-alpine
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
networks:
|
||||
- edge
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
s3-maintenance:
|
||||
image: mtusi-messenger-lab4-api:local
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-local}
|
||||
profiles: ["maintenance"]
|
||||
command: ["python", "-m", "src.app.reconcile_s3_orphans"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ROOT_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key_id
|
||||
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key
|
||||
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-86400}
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_root_access_key_id
|
||||
- s3_root_secret_access_key
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
restart: "no"
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
redis_password:
|
||||
file: ./secrets/redis_password
|
||||
s3_root_access_key_id:
|
||||
file: ./secrets/s3_root_access_key_id
|
||||
s3_root_secret_access_key:
|
||||
file: ./secrets/s3_root_secret_access_key
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
|
||||
networks:
|
||||
backend:
|
||||
internal: true
|
||||
edge:
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"oversized_upload": 413,
|
||||
"corrupt_image": 415,
|
||||
"mime_spoof": 415,
|
||||
"unsafe_filename": 400,
|
||||
"foreign_chat_attachment": 404,
|
||||
"unknown_attachment": 404,
|
||||
"result": "PASS"
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
# Результаты persistence и failover
|
||||
|
||||
Сводка составлена по выводу WSL, предоставленному автором. Hooks запущены через цели Makefile репозитория.
|
||||
|
||||
## Persistence: перезапуск хранилищ
|
||||
|
||||
Commande :
|
||||
|
||||
```sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080
|
||||
```
|
||||
|
||||
Результат: `PASS: persistence preserved confirmed data and existing session.`
|
||||
|
||||
## Failover одной API-реплики
|
||||
|
||||
Commande :
|
||||
|
||||
```sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080
|
||||
```
|
||||
|
||||
Résumé JSON affiché par le test :
|
||||
|
||||
```json
|
||||
{"observed_instances_before":2,"successful_writes":84,"failed_attempts":0,"first_recovery_seconds":1.578}
|
||||
```
|
||||
|
||||
Результат: `PASS: failover preserved confirmed data and existing session.`
|
||||
|
||||
## Восстановление после теста
|
||||
|
||||
Commande : `bash scripts/recover-api.sh`
|
||||
|
||||
Результат: `PASS: two API replicas are healthy through the load balancer.`
|
||||
|
||||
Тест также вывел `Hook scope needs operator evidence`. Эта сводка фиксирует результат тестового сценария, сообщённый автором; она не является независимым подтверждением состояния Docker.
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"liveness": 200,
|
||||
"readiness_during_outage": 503,
|
||||
"upload_during_outage": {
|
||||
"status": 503,
|
||||
"error_code": "unavailable",
|
||||
"elapsed_seconds": 9.187
|
||||
},
|
||||
"readiness_after_restore": 200,
|
||||
"result": "PASS"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
2026-11-13T22:46:05Z db93123e60d5
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"concurrent_clients": 10,
|
||||
"successful_uploads": 10,
|
||||
"decoded_dimensions": [
|
||||
3000,
|
||||
3000
|
||||
],
|
||||
"input_bytes": 35063,
|
||||
"container_memory_mib_before": {
|
||||
"db93123e60d5": 134.8,
|
||||
"bad2d3dd6806": 143.0
|
||||
},
|
||||
"container_memory_mib_peak": {
|
||||
"db93123e60d5": 150.2,
|
||||
"bad2d3dd6806": 151.0
|
||||
},
|
||||
"elapsed_seconds": 3.829,
|
||||
"observed_instances": [
|
||||
"bad2d3dd6806",
|
||||
"db93123e60d5"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
upstream messenger_api {
|
||||
zone messenger_api 64k;
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
keepalive 16;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
server_name _;
|
||||
|
||||
client_max_body_size 11m;
|
||||
client_body_buffer_size 64k;
|
||||
client_body_timeout 15s;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_read_timeout 10s;
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger_api;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 2s;
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY,
|
||||
username VARCHAR(32) NOT NULL UNIQUE
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name VARCHAR(100) NOT NULL
|
||||
CHECK (char_length(display_name) BETWEEN 1 AND 100 AND btrim(display_name) <> ''),
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title VARCHAR(100) NOT NULL
|
||||
CHECK (char_length(title) BETWEEN 1 AND 100 AND btrim(title) <> ''),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_id_idx ON chat_members(user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL,
|
||||
sender_id UUID NOT NULL,
|
||||
text VARCHAR(4000) NOT NULL
|
||||
CHECK (char_length(text) BETWEEN 1 AND 4000 AND btrim(text) <> ''),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT messages_sender_is_chat_member_fk
|
||||
FOREIGN KEY (chat_id, sender_id)
|
||||
REFERENCES chat_members(chat_id, user_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE idempotency_records (
|
||||
id UUID PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method VARCHAR(8) NOT NULL,
|
||||
path VARCHAR(200) NOT NULL,
|
||||
idempotency_key VARCHAR(128) NOT NULL,
|
||||
request_hash CHAR(64) NOT NULL,
|
||||
response_body JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
CONSTRAINT idempotency_records_scope_key_uq
|
||||
UNIQUE (user_id, method, path, idempotency_key),
|
||||
CHECK (method = 'POST'),
|
||||
CHECK (char_length(idempotency_key) BETWEEN 1 AND 128)
|
||||
);
|
||||
CREATE INDEX idempotency_records_expires_at_idx ON idempotency_records(expires_at);
|
||||
@@ -0,0 +1,48 @@
|
||||
-- Lab 4: file bytes remain in S3; PostgreSQL owns metadata and chat relations.
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_check;
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_text_length_ck CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
ALTER TABLE messages
|
||||
ADD CONSTRAINT messages_id_chat_id_uq UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL,
|
||||
owner_id UUID NOT NULL,
|
||||
filename VARCHAR(255) NOT NULL,
|
||||
content_type VARCHAR(32) NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'ready',
|
||||
variants JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
error_code VARCHAR(64),
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT attachments_owner_is_chat_member_fk
|
||||
FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members(chat_id, user_id) ON DELETE CASCADE,
|
||||
CONSTRAINT attachments_id_chat_id_uq UNIQUE (id, chat_id),
|
||||
CONSTRAINT attachments_filename_length_ck CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT attachments_content_type_ck CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size_bytes_ck CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256_ck CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_status_ck CHECK (status = 'ready')
|
||||
);
|
||||
CREATE INDEX attachments_chat_created_idx ON attachments(chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL,
|
||||
attachment_id UUID NOT NULL,
|
||||
chat_id UUID NOT NULL,
|
||||
position SMALLINT NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_message_chat_fk
|
||||
FOREIGN KEY (message_id, chat_id)
|
||||
REFERENCES messages(id, chat_id) ON DELETE CASCADE,
|
||||
CONSTRAINT message_attachments_attachment_chat_fk
|
||||
FOREIGN KEY (attachment_id, chat_id)
|
||||
REFERENCES attachments(id, chat_id) ON DELETE CASCADE,
|
||||
CONSTRAINT message_attachments_position_uq UNIQUE (message_id, position)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments(attachment_id);
|
||||
@@ -0,0 +1,30 @@
|
||||
-- Bring databases that already recorded an earlier 003_attachments layout
|
||||
-- forward without dropping attachment metadata or stored object references.
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS variants JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code VARCHAR(64);
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS width INTEGER;
|
||||
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS height INTEGER;
|
||||
ALTER TABLE attachments ALTER COLUMN width DROP NOT NULL;
|
||||
ALTER TABLE attachments ALTER COLUMN height DROP NOT NULL;
|
||||
|
||||
ALTER TABLE message_attachments ADD COLUMN IF NOT EXISTS chat_id UUID;
|
||||
UPDATE message_attachments AS link
|
||||
SET chat_id = message.chat_id
|
||||
FROM messages AS message
|
||||
WHERE message.id = link.message_id
|
||||
AND (link.chat_id IS NULL OR link.chat_id <> message.chat_id);
|
||||
ALTER TABLE message_attachments ALTER COLUMN chat_id SET NOT NULL;
|
||||
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_id_chat_id_uq;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_id_chat_id_uq UNIQUE (id, chat_id);
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_id_chat_id_uq;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_id_chat_id_uq UNIQUE (id, chat_id);
|
||||
|
||||
ALTER TABLE message_attachments DROP CONSTRAINT IF EXISTS message_attachments_message_chat_fk;
|
||||
ALTER TABLE message_attachments
|
||||
ADD CONSTRAINT message_attachments_message_chat_fk
|
||||
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE;
|
||||
ALTER TABLE message_attachments DROP CONSTRAINT IF EXISTS message_attachments_attachment_chat_fk;
|
||||
ALTER TABLE message_attachments
|
||||
ADD CONSTRAINT message_attachments_attachment_chat_fk
|
||||
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) ON DELETE CASCADE;
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
pydantic==2.9.2
|
||||
SQLAlchemy==2.0.36
|
||||
psycopg[binary]==3.2.3
|
||||
redis==5.2.0
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.44.1
|
||||
Pillow==13.0.0
|
||||
@@ -0,0 +1,57 @@
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$repo = Split-Path -Parent $PSScriptRoot
|
||||
$secretDir = Join-Path $repo 'secrets'
|
||||
New-Item -ItemType Directory -Path $secretDir -Force | Out-Null
|
||||
function New-Secret {
|
||||
$bytes = New-Object byte[] 32
|
||||
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
try { $rng.GetBytes($bytes) } finally { $rng.Dispose() }
|
||||
return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')
|
||||
}
|
||||
|
||||
function New-AccessKey {
|
||||
$alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
|
||||
$bytes = New-Object byte[] 20
|
||||
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
try { $rng.GetBytes($bytes) } finally { $rng.Dispose() }
|
||||
$value = -join ($bytes | ForEach-Object { $alphabet[($_ % $alphabet.Length)] })
|
||||
return $value
|
||||
}
|
||||
|
||||
function Read-Secret([string]$name) {
|
||||
$path = Join-Path $secretDir $name
|
||||
if (-not (Test-Path -LiteralPath $path)) { return $null }
|
||||
return [System.IO.File]::ReadAllText($path).Trim()
|
||||
}
|
||||
|
||||
function Write-Secret([string]$name, [string]$value) {
|
||||
$path = Join-Path $secretDir $name
|
||||
[System.IO.File]::WriteAllText($path, $value, [System.Text.UTF8Encoding]::new($false))
|
||||
}
|
||||
|
||||
function Ensure-Pair([string]$first, [string]$second, [scriptblock]$newFirst, [scriptblock]$newSecond) {
|
||||
$left = Read-Secret $first
|
||||
$right = Read-Secret $second
|
||||
if (($null -eq $left) -ne ($null -eq $right)) {
|
||||
throw "Only one file of credential pair '$first'/'$second' exists. Resolve it manually; credentials are never rotated automatically."
|
||||
}
|
||||
if ($null -eq $left) {
|
||||
$left = & $newFirst
|
||||
$right = & $newSecond $left
|
||||
Write-Secret $first $left
|
||||
Write-Secret $second $right
|
||||
}
|
||||
return @($left, $right)
|
||||
}
|
||||
|
||||
$pg = Ensure-Pair 'postgres_password' 'database_url' { New-Secret } {
|
||||
param($password)
|
||||
"postgresql+psycopg://messenger:$password@db:5432/messenger"
|
||||
}
|
||||
$redis = Ensure-Pair 'redis_password' 'cache_url' { New-Secret } {
|
||||
param($password)
|
||||
"redis://:$password@redis:6379/0"
|
||||
}
|
||||
$s3root = Ensure-Pair 's3_root_access_key_id' 's3_root_secret_access_key' { New-AccessKey } { New-Secret }
|
||||
$s3api = Ensure-Pair 's3_access_key_id' 's3_secret_access_key' { New-AccessKey } { New-Secret }
|
||||
Write-Output 'Local Postgres, Redis, RustFS root and limited API credentials are ready under secrets/. Existing complete pairs were left unchanged; values were not displayed.'
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
docker compose --env-file .env up -d --scale api=2 api lb >/dev/null
|
||||
deadline=$((SECONDS + 60))
|
||||
while (( SECONDS < deadline )); do
|
||||
healthy=0
|
||||
for container_id in $(docker compose --env-file .env ps -q api); do
|
||||
state=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id" 2>/dev/null || true)
|
||||
[[ "$state" == healthy ]] && healthy=$((healthy + 1))
|
||||
done
|
||||
if (( healthy >= 2 )) && curl --fail --silent http://localhost:8080/health/ready >/dev/null; then
|
||||
printf 'PASS: two API replicas are healthy through the load balancer.\n'
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
printf 'API replicas did not recover within 60 seconds.\n' >&2
|
||||
docker compose --env-file .env ps
|
||||
exit 1
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Stop the entry points before every state store; named volumes are left intact.
|
||||
docker compose stop lb api redis db s3
|
||||
docker compose start db redis s3
|
||||
|
||||
attempt=0
|
||||
until docker compose exec -T db pg_isready -U messenger -d messenger >/dev/null 2>&1 \
|
||||
&& docker compose exec -T redis sh -ec 'REDISCLI_AUTH="$(cat /run/secrets/redis_password)" redis-cli ping | grep -q PONG' >/dev/null 2>&1 \
|
||||
&& docker compose exec -T s3 curl -fsS http://127.0.0.1:9000/health/ready >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 60 ]; then
|
||||
echo 'Database/cache did not become ready within 60 seconds.' >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
docker compose --env-file .env up -d migrate s3-bootstrap
|
||||
docker compose --env-file .env up -d --scale api=2 api lb
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
|
||||
for container_id in $(docker compose ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
if [[ "$hostname" == "$TARGET_INSTANCE_ID" ]]; then
|
||||
mkdir -p evidence
|
||||
printf '%s %s\n' "$(date -u +%FT%TZ)" "$TARGET_INSTANCE_ID" >> evidence/stopped-replicas.txt
|
||||
docker stop --time 6 "$container_id" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
printf 'No running API container matches TARGET_INSTANCE_ID.\n' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1 @@
|
||||
"""MTUSI Lab 2 API."""
|
||||
@@ -0,0 +1,19 @@
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .settings import load_database_url
|
||||
|
||||
|
||||
engine = create_engine(
|
||||
load_database_url(),
|
||||
pool_size=5,
|
||||
max_overflow=5,
|
||||
pool_timeout=3,
|
||||
pool_recycle=1800,
|
||||
pool_pre_ping=True,
|
||||
connect_args={
|
||||
"connect_timeout": 3,
|
||||
"options": "-c statement_timeout=5000",
|
||||
},
|
||||
)
|
||||
SessionLocal = sessionmaker(bind=engine, expire_on_commit=False)
|
||||
@@ -0,0 +1,23 @@
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, code: str, message: str, status: int, challenge: str | None = None):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status = status
|
||||
self.challenge = challenge
|
||||
|
||||
|
||||
def error_response(request: Request, code: str, message: str, status: int, challenge: str | None = None):
|
||||
request_id = getattr(request.state, "request_id", "unknown")
|
||||
headers = {"X-Request-Id": request_id, "Cache-Control": "no-store"}
|
||||
if challenge:
|
||||
headers["WWW-Authenticate"] = challenge
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
headers=headers,
|
||||
content={"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
)
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Create 10,000 disposable rows and record the indexed keyset-page plan."""
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from .db import engine
|
||||
|
||||
|
||||
def run() -> None:
|
||||
user_id = uuid.uuid4()
|
||||
chat_id = uuid.uuid4()
|
||||
plan_json = None
|
||||
with engine.connect() as connection:
|
||||
transaction = connection.begin()
|
||||
try:
|
||||
connection.execute(
|
||||
text("INSERT INTO users(id, username, display_name, password_hash) "
|
||||
"VALUES (:id, :username, 'Synthetic plan fixture', 'not-a-login-hash')"),
|
||||
{"id": user_id, "username": "plan_" + uuid.uuid4().hex[:20]},
|
||||
)
|
||||
connection.execute(
|
||||
text("INSERT INTO chats(id, title) VALUES (:id, 'Synthetic plan fixture')"),
|
||||
{"id": chat_id},
|
||||
)
|
||||
connection.execute(
|
||||
text("INSERT INTO chat_members(chat_id, user_id) VALUES (:chat_id, :user_id)"),
|
||||
{"chat_id": chat_id, "user_id": user_id},
|
||||
)
|
||||
connection.execute(
|
||||
text("""
|
||||
INSERT INTO messages(id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), :chat_id, :sender_id,
|
||||
'synthetic message ' || n::text,
|
||||
TIMESTAMPTZ '2025-01-01 00:00:00+00' + n * INTERVAL '1 microsecond'
|
||||
FROM generate_series(1, 10000) AS n
|
||||
"""),
|
||||
{"chat_id": chat_id, "sender_id": user_id},
|
||||
)
|
||||
connection.exec_driver_sql("ANALYZE messages")
|
||||
raw = connection.execute(
|
||||
text("""
|
||||
EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = :chat_id
|
||||
AND (created_at, id) > (TIMESTAMPTZ '2025-01-01 00:00:00+00',
|
||||
'00000000-0000-0000-0000-000000000000'::uuid)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 50
|
||||
"""),
|
||||
{"chat_id": chat_id},
|
||||
).scalar_one()
|
||||
plan_json = raw if isinstance(raw, list) else json.loads(raw)
|
||||
finally:
|
||||
transaction.rollback()
|
||||
|
||||
document = plan_json[0]
|
||||
|
||||
def contains_history_index(node: dict) -> bool:
|
||||
if node.get("Index Name") == "messages_history_idx":
|
||||
return True
|
||||
return any(contains_history_index(child) for child in node.get("Plans", []))
|
||||
|
||||
if not contains_history_index(document["Plan"]):
|
||||
raise SystemExit("Planner did not use messages_history_idx; inspect and revise the query/index.")
|
||||
print(json.dumps(document, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
+818
@@ -0,0 +1,818 @@
|
||||
import base64
|
||||
import binascii
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.message import Message as EmailMessage
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import secrets
|
||||
import socket
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import uuid
|
||||
|
||||
from argon2 import PasswordHasher, Type
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
from fastapi import FastAPI, Header, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from redis import Redis
|
||||
from redis.exceptions import RedisError
|
||||
from sqlalchemy import and_, delete, func, or_, select, text, tuple_, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.exc import DBAPIError, OperationalError, TimeoutError as SQLAlchemyTimeoutError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from .db import SessionLocal, engine
|
||||
from .errors import ApiError, error_response
|
||||
from .models import Attachment, Chat, ChatMember, IdempotencyRecord, Message, MessageAttachment, User
|
||||
from .schemas import (
|
||||
AttachmentOut,
|
||||
ChatOut,
|
||||
ChatPage,
|
||||
CreateChat,
|
||||
CreateMessage,
|
||||
CreateUser,
|
||||
Health,
|
||||
MessageOut,
|
||||
MessagePage,
|
||||
SessionOut,
|
||||
UserOut,
|
||||
)
|
||||
from .settings import load_settings
|
||||
from .storage import s3
|
||||
|
||||
settings = load_settings()
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
)
|
||||
logger = logging.getLogger("messenger.api")
|
||||
password_hasher = PasswordHasher(
|
||||
time_cost=2,
|
||||
memory_cost=19_456,
|
||||
parallelism=1,
|
||||
hash_len=32,
|
||||
salt_len=16,
|
||||
type=Type.ID,
|
||||
)
|
||||
DUMMY_PASSWORD_HASH = password_hasher.hash("dummy-password-for-timing-only")
|
||||
redis_client = Redis.from_url(
|
||||
settings.cache_url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=2,
|
||||
socket_timeout=2,
|
||||
health_check_interval=30,
|
||||
)
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger — Lab 4", version="1.4.0")
|
||||
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
TOKEN_RE = re.compile(r"^[A-Za-z0-9_-]{43}$")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII)
|
||||
# Docker assigns each container a stable hostname equal to its container ID.
|
||||
# The load balancer forwards this value so tests can prove requests changed replicas.
|
||||
INSTANCE_ID = socket.gethostname()
|
||||
UPLOAD_LIMITER = asyncio.Semaphore(1)
|
||||
|
||||
|
||||
def _iso(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _user_out(user: User) -> dict:
|
||||
return {
|
||||
"id": str(user.id),
|
||||
"username": user.username,
|
||||
"display_name": user.display_name,
|
||||
"created_at": _iso(user.created_at),
|
||||
}
|
||||
|
||||
|
||||
def _message_out(message: Message, attachment_ids: list[uuid.UUID | str] | None = None) -> dict:
|
||||
return {
|
||||
"id": str(message.id),
|
||||
"chat_id": str(message.chat_id),
|
||||
"sender_id": str(message.sender_id),
|
||||
"text": message.text,
|
||||
"created_at": _iso(message.created_at),
|
||||
"attachment_ids": [str(item) for item in (attachment_ids or [])],
|
||||
}
|
||||
|
||||
|
||||
def _attachment_out(attachment: Attachment) -> dict:
|
||||
return {
|
||||
"id": str(attachment.id),
|
||||
"chat_id": str(attachment.chat_id),
|
||||
"owner_id": str(attachment.owner_id),
|
||||
"filename": attachment.filename,
|
||||
"content_type": attachment.content_type,
|
||||
"size_bytes": attachment.size_bytes,
|
||||
"sha256": attachment.sha256,
|
||||
"status": attachment.status,
|
||||
"variants": attachment.variants or [],
|
||||
"error_code": attachment.error_code,
|
||||
"created_at": _iso(attachment.created_at),
|
||||
"updated_at": _iso(attachment.updated_at),
|
||||
}
|
||||
|
||||
|
||||
def _message_attachment_ids(session, message_id: uuid.UUID) -> list[uuid.UUID]:
|
||||
return session.scalars(
|
||||
select(MessageAttachment.attachment_id)
|
||||
.where(MessageAttachment.message_id == message_id)
|
||||
.order_by(MessageAttachment.position)
|
||||
).all()
|
||||
|
||||
|
||||
def _validate_image(file_obj, claimed_content_type: str) -> str:
|
||||
file_obj.seek(0)
|
||||
try:
|
||||
with Image.open(file_obj) as image:
|
||||
detected = {
|
||||
"PNG": "image/png",
|
||||
"JPEG": "image/jpeg",
|
||||
}.get(image.format or "")
|
||||
if detected is None or detected != claimed_content_type:
|
||||
raise _api_error("unsupported_media_type", "Формат изображения не поддерживается", 415)
|
||||
width, height = image.size
|
||||
if width < 1 or height < 1 or width * height > settings.max_image_pixels:
|
||||
raise _api_error("payload_too_large", "Размер изображения превышает допустимый предел", 413)
|
||||
image.verify()
|
||||
file_obj.seek(0)
|
||||
with Image.open(file_obj) as image:
|
||||
image.load()
|
||||
except ApiError:
|
||||
raise
|
||||
except (UnidentifiedImageError, OSError, SyntaxError, ValueError, Image.DecompressionBombError):
|
||||
raise _api_error("unsupported_media_type", "Изображение повреждено или не поддерживается", 415) from None
|
||||
finally:
|
||||
file_obj.seek(0)
|
||||
return claimed_content_type
|
||||
|
||||
|
||||
def _valid_filename(filename: str) -> bool:
|
||||
if not 1 <= len(filename) <= 255:
|
||||
return False
|
||||
if "/" in filename or "\\" in filename:
|
||||
return False
|
||||
return not any(unicodedata.category(char) == "Cc" for char in filename)
|
||||
|
||||
|
||||
def _is_not_found_s3(exc: ClientError) -> bool:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
return code in {"404", "NoSuchKey", "NoSuchBucket", "NotFound"}
|
||||
|
||||
|
||||
def _chat_out(session, chat: Chat) -> dict:
|
||||
members = session.scalars(
|
||||
select(ChatMember.user_id)
|
||||
.where(ChatMember.chat_id == chat.id)
|
||||
.order_by(ChatMember.user_id)
|
||||
).all()
|
||||
return {
|
||||
"id": str(chat.id),
|
||||
"title": chat.title,
|
||||
"member_ids": [str(member_id) for member_id in members],
|
||||
"created_at": _iso(chat.created_at),
|
||||
}
|
||||
|
||||
|
||||
def _api_error(code: str, message: str, status: int, challenge: str | None = None) -> ApiError:
|
||||
return ApiError(code, message, status, challenge)
|
||||
|
||||
|
||||
def _unauthorized(challenge: str = BEARER_CHALLENGE) -> ApiError:
|
||||
return _api_error("unauthorized", "Нет действительной идентификации", 401, challenge)
|
||||
|
||||
|
||||
def _encode_cursor(payload: dict) -> str:
|
||||
raw = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def _decode_cursor(value: str) -> dict:
|
||||
try:
|
||||
padded = value + ("=" * ((4 - len(value) % 4) % 4))
|
||||
raw = base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True)
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("cursor must be an object")
|
||||
return payload
|
||||
except (ValueError, UnicodeError, binascii.Error, json.JSONDecodeError) as exc:
|
||||
raise _api_error("invalid_request", "Неверный cursor", 400) from exc
|
||||
|
||||
|
||||
def _cursor_position(payload: dict) -> tuple[datetime, uuid.UUID]:
|
||||
try:
|
||||
created_at = datetime.fromisoformat(payload["t"].replace("Z", "+00:00"))
|
||||
if created_at.tzinfo is None:
|
||||
raise ValueError("cursor time must include timezone")
|
||||
item_id = uuid.UUID(payload["i"])
|
||||
return created_at.astimezone(timezone.utc), item_id
|
||||
except (KeyError, TypeError, ValueError, AttributeError) as exc:
|
||||
raise _api_error("invalid_request", "Неверный cursor", 400) from exc
|
||||
|
||||
|
||||
def _session_key(token: str) -> str:
|
||||
digest = hashlib.sha256(token.encode("ascii")).hexdigest()
|
||||
return "session:v1:" + digest
|
||||
|
||||
|
||||
def _parse_bearer(request: Request) -> str:
|
||||
header = request.headers.get("authorization", "")
|
||||
scheme, separator, token = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not separator or not TOKEN_RE.fullmatch(token):
|
||||
raise _unauthorized()
|
||||
return token
|
||||
|
||||
|
||||
def _load_session(token: str) -> dict:
|
||||
value = redis_client.get(_session_key(token))
|
||||
if value is None:
|
||||
raise _unauthorized()
|
||||
try:
|
||||
session = json.loads(value)
|
||||
user_id = uuid.UUID(session["user_id"])
|
||||
expires_at = datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00"))
|
||||
if expires_at.tzinfo is None or datetime.now(timezone.utc) >= expires_at:
|
||||
redis_client.delete(_session_key(token))
|
||||
raise _unauthorized()
|
||||
return {"user_id": user_id, "expires_at": expires_at}
|
||||
except ApiError:
|
||||
raise
|
||||
except (KeyError, TypeError, ValueError, AttributeError, json.JSONDecodeError):
|
||||
redis_client.delete(_session_key(token))
|
||||
raise _unauthorized() from None
|
||||
|
||||
|
||||
def _current_user(request: Request) -> tuple[User, str]:
|
||||
token = _parse_bearer(request)
|
||||
session_data = _load_session(token)
|
||||
with SessionLocal() as session:
|
||||
user = session.get(User, session_data["user_id"])
|
||||
if user is None:
|
||||
redis_client.delete(_session_key(token))
|
||||
raise _unauthorized()
|
||||
return user, token
|
||||
|
||||
|
||||
def _parse_basic(header: str) -> tuple[str, str]:
|
||||
scheme, separator, encoded = header.partition(" ")
|
||||
if scheme.lower() != "basic" or not separator or not encoded:
|
||||
raise _unauthorized(BASIC_CHALLENGE)
|
||||
try:
|
||||
decoded = base64.b64decode(encoded.encode("ascii"), validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
except (UnicodeError, ValueError, binascii.Error):
|
||||
raise _unauthorized(BASIC_CHALLENGE) from None
|
||||
if not USERNAME_RE.fullmatch(username) or not password:
|
||||
raise _unauthorized(BASIC_CHALLENGE)
|
||||
return username, password
|
||||
|
||||
|
||||
def _password_matches(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return password_hasher.verify(password_hash, password)
|
||||
except (VerifyMismatchError, VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def _create_session(username: str, password: str) -> dict:
|
||||
with SessionLocal() as session:
|
||||
user = session.scalar(select(User).where(User.username == username))
|
||||
stored_hash = user.password_hash if user is not None else DUMMY_PASSWORD_HASH
|
||||
password_ok = _password_matches(stored_hash, password)
|
||||
if user is None or not password_ok:
|
||||
raise _unauthorized(BASIC_CHALLENGE)
|
||||
user_payload = _user_out(user)
|
||||
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.session_ttl_seconds)
|
||||
for _ in range(3):
|
||||
token = secrets.token_urlsafe(32)
|
||||
stored_value = json.dumps(
|
||||
{"user_id": user_payload["id"], "expires_at": _iso(expires_at)},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
if redis_client.set(_session_key(token), stored_value, ex=settings.session_ttl_seconds, nx=True):
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": _iso(expires_at),
|
||||
"user": user_payload,
|
||||
}
|
||||
logger.error("session_token_collision_limit_reached")
|
||||
raise _api_error("unavailable", "Сервис временно недоступен", 503)
|
||||
|
||||
|
||||
def _get_chat_for_member(session, chat_id: uuid.UUID, user_id: uuid.UUID) -> Chat | None:
|
||||
return session.scalar(
|
||||
select(Chat)
|
||||
.join(ChatMember, ChatMember.chat_id == Chat.id)
|
||||
.where(Chat.id == chat_id, ChatMember.user_id == user_id)
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_metadata_middleware(request: Request, call_next):
|
||||
request_id = str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception as exc:
|
||||
# Keep an unexpected failure bounded and ensure the API response still
|
||||
# identifies the request and the replica that created it.
|
||||
logger.error("internal_error request_id=%s type=%s", request_id, type(exc).__name__)
|
||||
response = error_response(request, "internal_error", "Внутренняя ошибка", 500)
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def handle_api_error(request: Request, exc: ApiError):
|
||||
return error_response(request, exc.code, exc.message, exc.status, exc.challenge)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def handle_validation_error(request: Request, _exc: RequestValidationError):
|
||||
return error_response(request, "invalid_request", "Неверный запрос", 400)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def handle_http_error(request: Request, exc: StarletteHTTPException):
|
||||
if exc.status_code == 404:
|
||||
return error_response(request, "not_found", "Объект не найден", 404)
|
||||
return error_response(request, "invalid_request", "Неверный запрос", 400)
|
||||
|
||||
|
||||
@app.exception_handler(OperationalError)
|
||||
@app.exception_handler(SQLAlchemyTimeoutError)
|
||||
async def handle_database_unavailable(request: Request, exc: Exception):
|
||||
logger.warning("database_unavailable request_id=%s type=%s", request.state.request_id, type(exc).__name__)
|
||||
return error_response(request, "unavailable", "Сервис временно недоступен", 503)
|
||||
|
||||
|
||||
@app.exception_handler(RedisError)
|
||||
async def handle_cache_unavailable(request: Request, exc: RedisError):
|
||||
logger.warning("session_store_unavailable request_id=%s type=%s", request.state.request_id, type(exc).__name__)
|
||||
return error_response(request, "unavailable", "Сервис временно недоступен", 503)
|
||||
|
||||
|
||||
@app.exception_handler(ClientError)
|
||||
@app.exception_handler(BotoCoreError)
|
||||
async def handle_object_store_unavailable(request: Request, exc: Exception):
|
||||
logger.warning("object_store_unavailable request_id=%s type=%s", request.state.request_id, type(exc).__name__)
|
||||
return error_response(request, "unavailable", "Сервис временно недоступен", 503)
|
||||
|
||||
|
||||
@app.exception_handler(DBAPIError)
|
||||
async def handle_database_error(request: Request, exc: DBAPIError):
|
||||
if exc.connection_invalidated:
|
||||
logger.warning("database_connection_lost request_id=%s", request.state.request_id)
|
||||
return error_response(request, "unavailable", "Сервис временно недоступен", 503)
|
||||
logger.error("database_error request_id=%s type=%s", request.state.request_id, type(exc).__name__)
|
||||
return error_response(request, "internal_error", "Внутренняя ошибка", 500)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def handle_unexpected_error(request: Request, exc: Exception):
|
||||
logger.error("internal_error request_id=%s type=%s", request.state.request_id, type(exc).__name__)
|
||||
return error_response(request, "internal_error", "Внутренняя ошибка", 500)
|
||||
|
||||
|
||||
@app.get("/health/live", response_model=Health)
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready", response_model=Health, responses={503: {"description": "Dependency unavailable"}})
|
||||
def ready():
|
||||
with SessionLocal() as session:
|
||||
session.execute(text("SELECT 1"))
|
||||
redis_client.ping()
|
||||
s3.get_bucket_location(Bucket=settings.s3_bucket)
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", response_model=UserOut, status_code=201)
|
||||
def register_user(payload: CreateUser):
|
||||
password_hash = password_hasher.hash(payload.password)
|
||||
user = User(username=payload.username, display_name=payload.display_name, password_hash=password_hash)
|
||||
try:
|
||||
with SessionLocal.begin() as session:
|
||||
session.add(user)
|
||||
session.flush()
|
||||
except DBAPIError as exc:
|
||||
if getattr(getattr(exc, "orig", None), "sqlstate", None) == "23505":
|
||||
raise _api_error("conflict", "username занят", 409) from exc
|
||||
raise
|
||||
return _user_out(user)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", response_model=SessionOut, status_code=201)
|
||||
async def create_session(request: Request):
|
||||
if await request.body():
|
||||
raise _api_error("invalid_request", "Basic authentication must not include a body", 400)
|
||||
username, password = _parse_basic(request.headers.get("authorization", ""))
|
||||
return await run_in_threadpool(_create_session, username, password)
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def revoke_session(request: Request):
|
||||
_user, token = _current_user(request)
|
||||
redis_client.delete(_session_key(token))
|
||||
return None
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me", response_model=UserOut)
|
||||
def current_user(request: Request):
|
||||
user, _token = _current_user(request)
|
||||
return _user_out(user)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", response_model=ChatOut, status_code=201)
|
||||
def create_chat(payload: CreateChat, request: Request):
|
||||
user, _token = _current_user(request)
|
||||
member_ids = [uuid.UUID(value) for value in payload.member_ids]
|
||||
if user.id not in member_ids:
|
||||
raise _api_error("invalid_request", "Вызывающий должен входить в member_ids", 400)
|
||||
|
||||
try:
|
||||
with SessionLocal.begin() as session:
|
||||
chat = Chat(title=payload.title)
|
||||
session.add(chat)
|
||||
session.flush()
|
||||
session.add_all(ChatMember(chat_id=chat.id, user_id=member_id) for member_id in member_ids)
|
||||
session.flush()
|
||||
result = _chat_out(session, chat)
|
||||
except DBAPIError as exc:
|
||||
if getattr(getattr(exc, "orig", None), "sqlstate", None) in {"23503", "23505"}:
|
||||
raise _api_error("invalid_request", "Неизвестный участник", 400) from exc
|
||||
raise
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/api/v1/chats", response_model=ChatPage)
|
||||
def list_chats(request: Request, limit: int = 50, cursor: str | None = None):
|
||||
user, _token = _current_user(request)
|
||||
if limit < 1 or limit > 100:
|
||||
raise _api_error("invalid_request", "limit вне диапазона", 400)
|
||||
statement = (
|
||||
select(Chat)
|
||||
.join(ChatMember, ChatMember.chat_id == Chat.id)
|
||||
.where(ChatMember.user_id == user.id)
|
||||
.order_by(Chat.created_at, Chat.id)
|
||||
)
|
||||
if cursor is not None:
|
||||
payload = _decode_cursor(cursor)
|
||||
if payload.get("u") != str(user.id):
|
||||
raise _api_error("invalid_request", "Неверный cursor", 400)
|
||||
created_at, chat_id = _cursor_position(payload)
|
||||
statement = statement.where(tuple_(Chat.created_at, Chat.id) > tuple_(created_at, chat_id))
|
||||
|
||||
with SessionLocal() as session:
|
||||
chats = session.scalars(statement.limit(limit + 1)).all()
|
||||
has_more = len(chats) > limit
|
||||
page = chats[:limit]
|
||||
items = [_chat_out(session, chat) for chat in page]
|
||||
next_cursor = None
|
||||
if has_more:
|
||||
last = page[-1]
|
||||
next_cursor = _encode_cursor({"u": str(user.id), "t": _iso(last.created_at), "i": str(last.id)})
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}", response_model=ChatOut)
|
||||
def get_chat(chat_id: uuid.UUID, request: Request):
|
||||
user, _token = _current_user(request)
|
||||
with SessionLocal() as session:
|
||||
chat = _get_chat_for_member(session, chat_id, user.id)
|
||||
if chat is None:
|
||||
raise _api_error("not_found", "Объект не найден", 404)
|
||||
return _chat_out(session, chat)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", response_model=MessageOut, status_code=201)
|
||||
def create_message(
|
||||
chat_id: uuid.UUID,
|
||||
payload: CreateMessage,
|
||||
request: Request,
|
||||
idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"),
|
||||
):
|
||||
user, _token = _current_user(request)
|
||||
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||
raise _api_error("invalid_request", "Нужен корректный Idempotency-Key", 400)
|
||||
if not payload.text.strip() and not payload.attachment_ids:
|
||||
raise _api_error("invalid_request", "Сообщение не может состоять только из пробелов", 400)
|
||||
|
||||
# Hash the validated semantic JSON, not the raw bytes, so whitespace and
|
||||
# object key order do not change the identity of a request.
|
||||
canonical_body = json.dumps(
|
||||
{"text": payload.text, "attachment_ids": payload.attachment_ids},
|
||||
ensure_ascii=False, sort_keys=True, separators=(",", ":"),
|
||||
)
|
||||
body_hash = hashlib.sha256(canonical_body.encode("utf-8")).hexdigest()
|
||||
legacy_hash = hashlib.sha256(
|
||||
json.dumps({"text": payload.text}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
path = request.url.path
|
||||
now = datetime.now(timezone.utc)
|
||||
expires_at = now + timedelta(hours=24)
|
||||
scope = (
|
||||
IdempotencyRecord.user_id == user.id,
|
||||
IdempotencyRecord.method == "POST",
|
||||
IdempotencyRecord.path == path,
|
||||
IdempotencyRecord.idempotency_key == idempotency_key,
|
||||
)
|
||||
|
||||
with SessionLocal.begin() as session:
|
||||
chat = _get_chat_for_member(session, chat_id, user.id)
|
||||
if chat is None:
|
||||
raise _api_error("not_found", "Объект не найден", 404)
|
||||
attachment_ids = [uuid.UUID(value) for value in payload.attachment_ids]
|
||||
if attachment_ids:
|
||||
attached = session.scalars(
|
||||
select(Attachment).where(
|
||||
Attachment.chat_id == chat_id,
|
||||
Attachment.id.in_(attachment_ids),
|
||||
)
|
||||
).all()
|
||||
if len(attached) != len(attachment_ids):
|
||||
raise _api_error("not_found", "Объект не найден", 404)
|
||||
if any(item.status != "ready" for item in attached):
|
||||
raise _api_error("conflict", "Вложение ещё не готово", 409)
|
||||
|
||||
# A key becomes reusable only after its 24-hour retention window.
|
||||
session.execute(delete(IdempotencyRecord).where(*scope, IdempotencyRecord.expires_at <= now))
|
||||
insert_record = (
|
||||
pg_insert(IdempotencyRecord)
|
||||
.values(
|
||||
user_id=user.id,
|
||||
method="POST",
|
||||
path=path,
|
||||
idempotency_key=idempotency_key,
|
||||
request_hash=body_hash,
|
||||
response_body={},
|
||||
expires_at=expires_at,
|
||||
)
|
||||
.on_conflict_do_nothing(constraint="idempotency_records_scope_key_uq")
|
||||
.returning(IdempotencyRecord.id)
|
||||
)
|
||||
record_id = session.execute(insert_record).scalar_one_or_none()
|
||||
|
||||
if record_id is None:
|
||||
# A concurrent insert waits on the unique constraint. At READ
|
||||
# COMMITTED the winning transaction is visible to this SELECT.
|
||||
existing = session.scalar(select(IdempotencyRecord).where(*scope))
|
||||
if existing is None:
|
||||
raise _api_error("unavailable", "Сервис временно недоступен", 503)
|
||||
if existing.request_hash != body_hash and not (not attachment_ids and existing.request_hash == legacy_hash):
|
||||
raise _api_error("conflict", "Idempotency-Key уже использован для другого запроса", 409)
|
||||
result = dict(existing.response_body)
|
||||
result.setdefault("attachment_ids", [])
|
||||
else:
|
||||
message = Message(chat_id=chat_id, sender_id=user.id, text=payload.text)
|
||||
session.add(message)
|
||||
session.flush()
|
||||
session.add_all(
|
||||
MessageAttachment(
|
||||
message_id=message.id,
|
||||
attachment_id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
position=position,
|
||||
)
|
||||
for position, attachment_id in enumerate(attachment_ids)
|
||||
)
|
||||
result = _message_out(message, attachment_ids)
|
||||
session.execute(
|
||||
update(IdempotencyRecord)
|
||||
.where(IdempotencyRecord.id == record_id)
|
||||
.values(response_body=result)
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
|
||||
def list_messages(chat_id: uuid.UUID, request: Request, limit: int = 50, cursor: str | None = None):
|
||||
user, _token = _current_user(request)
|
||||
with SessionLocal() as session:
|
||||
chat = _get_chat_for_member(session, chat_id, user.id)
|
||||
if chat is None:
|
||||
raise _api_error("not_found", "Объект не найден", 404)
|
||||
if limit < 1 or limit > 100:
|
||||
raise _api_error("invalid_request", "limit вне диапазона", 400)
|
||||
after = None
|
||||
if cursor is not None:
|
||||
payload = _decode_cursor(cursor)
|
||||
if payload.get("c") != str(chat_id):
|
||||
raise _api_error("invalid_request", "cursor от другого чата", 400)
|
||||
after = _cursor_position(payload)
|
||||
statement = select(Message).where(Message.chat_id == chat_id)
|
||||
if after is not None:
|
||||
statement = statement.where(tuple_(Message.created_at, Message.id) > tuple_(*after))
|
||||
statement = statement.order_by(Message.created_at, Message.id).limit(limit + 1)
|
||||
rows = session.scalars(statement).all()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
attachment_rows = session.execute(
|
||||
select(MessageAttachment.message_id, MessageAttachment.attachment_id)
|
||||
.where(MessageAttachment.message_id.in_([message.id for message in page]))
|
||||
.order_by(MessageAttachment.message_id, MessageAttachment.position)
|
||||
).all() if page else []
|
||||
ids_by_message: dict[uuid.UUID, list[uuid.UUID]] = {}
|
||||
for message_id, attachment_id in attachment_rows:
|
||||
ids_by_message.setdefault(message_id, []).append(attachment_id)
|
||||
items = [_message_out(message, ids_by_message.get(message.id, [])) for message in page]
|
||||
|
||||
next_cursor = None
|
||||
if has_more:
|
||||
last = page[-1]
|
||||
next_cursor = _encode_cursor({"c": str(chat_id), "t": _iso(last.created_at), "i": str(last.id)})
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
response_model=AttachmentOut,
|
||||
status_code=201,
|
||||
)
|
||||
async def upload_attachment(
|
||||
chat_id: uuid.UUID,
|
||||
request: Request,
|
||||
filename: str,
|
||||
):
|
||||
user, _token = _current_user(request)
|
||||
with SessionLocal() as session:
|
||||
if _get_chat_for_member(session, chat_id, user.id) is None:
|
||||
raise _api_error("not_found", "Объект не найден", 404)
|
||||
|
||||
if not _valid_filename(filename):
|
||||
raise _api_error("invalid_request", "Неверное имя файла", 400)
|
||||
content_type_header = request.headers.get("content-type", "")
|
||||
header = EmailMessage()
|
||||
try:
|
||||
header["Content-Type"] = content_type_header
|
||||
claimed_type = header.get_content_type().lower()
|
||||
except (TypeError, ValueError):
|
||||
claimed_type = ""
|
||||
if claimed_type not in {"image/png", "image/jpeg"}:
|
||||
raise _api_error("unsupported_media_type", "Поддерживаются только PNG и JPEG", 415)
|
||||
content_length = request.headers.get("content-length")
|
||||
if content_length is not None:
|
||||
try:
|
||||
if int(content_length) > settings.max_upload_bytes:
|
||||
raise _api_error("payload_too_large", "Файл превышает допустимый размер", 413)
|
||||
except ValueError:
|
||||
raise _api_error("invalid_request", "Неверный Content-Length", 400) from None
|
||||
|
||||
async with UPLOAD_LIMITER:
|
||||
with tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b", dir="/tmp") as temporary:
|
||||
size_bytes = 0
|
||||
digest = hashlib.sha256()
|
||||
async for chunk in request.stream():
|
||||
if not chunk:
|
||||
continue
|
||||
size_bytes += len(chunk)
|
||||
if size_bytes > settings.max_upload_bytes:
|
||||
raise _api_error("payload_too_large", "Файл превышает допустимый размер", 413)
|
||||
digest.update(chunk)
|
||||
temporary.write(chunk)
|
||||
if size_bytes == 0:
|
||||
raise _api_error("unsupported_media_type", "Файл пуст", 415)
|
||||
|
||||
detected_type = await run_in_threadpool(_validate_image, temporary, claimed_type)
|
||||
attachment_id = uuid.uuid4()
|
||||
object_key = f"originals/{attachment_id.hex}"
|
||||
temporary.seek(0)
|
||||
try:
|
||||
await run_in_threadpool(
|
||||
s3.put_object,
|
||||
Bucket=settings.s3_bucket,
|
||||
Key=object_key,
|
||||
Body=temporary,
|
||||
ContentLength=size_bytes,
|
||||
ContentType=detected_type,
|
||||
Metadata={"sha256": digest.hexdigest()},
|
||||
)
|
||||
except (ClientError, BotoCoreError):
|
||||
try:
|
||||
await run_in_threadpool(s3.delete_object, Bucket=settings.s3_bucket, Key=object_key)
|
||||
except Exception as cleanup_exc:
|
||||
logger.warning(
|
||||
"attachment_compensation_failed request_id=%s type=%s",
|
||||
request.state.request_id,
|
||||
type(cleanup_exc).__name__,
|
||||
)
|
||||
raise _api_error("unavailable", "Сервис временно недоступен", 503) from None
|
||||
|
||||
try:
|
||||
with SessionLocal.begin() as session:
|
||||
attachment = Attachment(
|
||||
id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=user.id,
|
||||
filename=filename,
|
||||
content_type=detected_type,
|
||||
size_bytes=size_bytes,
|
||||
sha256=digest.hexdigest(),
|
||||
status="ready",
|
||||
variants=[],
|
||||
error_code=None,
|
||||
object_key=object_key,
|
||||
)
|
||||
session.add(attachment)
|
||||
session.flush()
|
||||
result = _attachment_out(attachment)
|
||||
except Exception:
|
||||
try:
|
||||
await run_in_threadpool(s3.delete_object, Bucket=settings.s3_bucket, Key=object_key)
|
||||
except Exception as cleanup_exc:
|
||||
logger.warning(
|
||||
"attachment_compensation_failed request_id=%s type=%s",
|
||||
request.state.request_id,
|
||||
type(cleanup_exc).__name__,
|
||||
)
|
||||
raise
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentOut)
|
||||
def get_attachment(attachment_id: uuid.UUID, request: Request):
|
||||
user, _token = _current_user(request)
|
||||
with SessionLocal() as session:
|
||||
attachment = session.scalar(
|
||||
select(Attachment)
|
||||
.join(ChatMember, ChatMember.chat_id == Attachment.chat_id)
|
||||
.where(Attachment.id == attachment_id, ChatMember.user_id == user.id)
|
||||
)
|
||||
if attachment is None:
|
||||
raise _api_error("not_found", "Объект не найден", 404)
|
||||
return _attachment_out(attachment)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def download_attachment(
|
||||
attachment_id: uuid.UUID,
|
||||
request: Request,
|
||||
variant: str = "original",
|
||||
):
|
||||
user, _token = _current_user(request)
|
||||
with SessionLocal() as session:
|
||||
attachment = session.scalar(
|
||||
select(Attachment)
|
||||
.join(ChatMember, ChatMember.chat_id == Attachment.chat_id)
|
||||
.where(Attachment.id == attachment_id, ChatMember.user_id == user.id)
|
||||
)
|
||||
if attachment is None:
|
||||
raise _api_error("not_found", "Объект не найден", 404)
|
||||
if attachment.status != "ready":
|
||||
raise _api_error("conflict", "Вложение ещё не готово", 409)
|
||||
if variant != "original":
|
||||
raise _api_error("not_found", "Объект не найден", 404)
|
||||
object_key = attachment.object_key
|
||||
content_type = attachment.content_type
|
||||
size_bytes = attachment.size_bytes
|
||||
|
||||
try:
|
||||
stored = await run_in_threadpool(s3.get_object, Bucket=settings.s3_bucket, Key=object_key)
|
||||
except ClientError as exc:
|
||||
if _is_not_found_s3(exc):
|
||||
logger.error("attachment_object_missing request_id=%s", request.state.request_id)
|
||||
raise _api_error("unavailable", "Сервис временно недоступен", 503) from None
|
||||
raise _api_error("unavailable", "Сервис временно недоступен", 503) from None
|
||||
except BotoCoreError:
|
||||
raise _api_error("unavailable", "Сервис временно недоступен", 503) from None
|
||||
|
||||
body = stored["Body"]
|
||||
|
||||
async def chunks():
|
||||
try:
|
||||
while True:
|
||||
chunk = await run_in_threadpool(body.read, 64 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
finally:
|
||||
await run_in_threadpool(body.close)
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(),
|
||||
media_type=content_type,
|
||||
headers={
|
||||
"Content-Length": str(size_bytes),
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,36 @@
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from .db import engine
|
||||
|
||||
|
||||
MIGRATIONS = Path(__file__).resolve().parents[2] / "migrations"
|
||||
|
||||
|
||||
def migrate() -> None:
|
||||
with engine.begin() as connection:
|
||||
connection.exec_driver_sql("SELECT pg_advisory_xact_lock(2026102202)")
|
||||
connection.exec_driver_sql(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(version VARCHAR(128) PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP)"
|
||||
)
|
||||
applied = set(connection.execute(text("SELECT version FROM schema_migrations")).scalars())
|
||||
for path in sorted(MIGRATIONS.glob("*.sql")):
|
||||
version = path.stem
|
||||
if version in applied:
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
for statement in sql.split(";"):
|
||||
if statement.strip():
|
||||
connection.exec_driver_sql(statement)
|
||||
connection.execute(
|
||||
text("INSERT INTO schema_migrations(version) VALUES (:version)"),
|
||||
{"version": version},
|
||||
)
|
||||
print(f"Applied migration {version}")
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
migrate()
|
||||
@@ -0,0 +1,145 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, ForeignKeyConstraint, Index, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB, UUID
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
username: Mapped[str] = mapped_column(String(32), nullable=False, unique=True)
|
||||
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
__tablename__ = "chats"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
title: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
|
||||
|
||||
class ChatMember(Base):
|
||||
__tablename__ = "chat_members"
|
||||
__table_args__ = (
|
||||
Index("chat_members_user_id_idx", "user_id", "chat_id"),
|
||||
)
|
||||
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("users.id", ondelete="RESTRICT"), primary_key=True
|
||||
)
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = "messages"
|
||||
__table_args__ = (
|
||||
ForeignKeyConstraint(
|
||||
["chat_id", "sender_id"],
|
||||
["chat_members.chat_id", "chat_members.user_id"],
|
||||
name="messages_sender_is_chat_member_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
UniqueConstraint("id", "chat_id", name="messages_id_chat_id_uq"),
|
||||
Index("messages_history_idx", "chat_id", "created_at", "id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
|
||||
sender_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
|
||||
text: Mapped[str] = mapped_column(String(4000), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
|
||||
|
||||
class IdempotencyRecord(Base):
|
||||
"""Shared, transactional result cache for retried message POST requests."""
|
||||
|
||||
__tablename__ = "idempotency_records"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"user_id", "method", "path", "idempotency_key",
|
||||
name="idempotency_records_scope_key_uq",
|
||||
),
|
||||
Index("idempotency_records_expires_at_idx", "expires_at"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
method: Mapped[str] = mapped_column(String(8), nullable=False)
|
||||
path: Mapped[str] = mapped_column(String(200), nullable=False)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
response_body: Mapped[dict] = mapped_column(JSONB, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
__tablename__ = "attachments"
|
||||
__table_args__ = (
|
||||
ForeignKeyConstraint(
|
||||
["chat_id", "owner_id"],
|
||||
["chat_members.chat_id", "chat_members.user_id"],
|
||||
name="attachments_owner_is_chat_member_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
UniqueConstraint("id", "chat_id", name="attachments_id_chat_id_uq"),
|
||||
CheckConstraint("char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_length_ck"),
|
||||
CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_ck"),
|
||||
CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_ck"),
|
||||
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="attachments_sha256_ck"),
|
||||
CheckConstraint("status = 'ready'", name="attachments_status_ck"),
|
||||
Index("attachments_chat_created_idx", "chat_id", "created_at", "id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
|
||||
owner_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
|
||||
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="ready")
|
||||
variants: Mapped[list] = mapped_column(JSONB, nullable=False, default=list, server_default="[]")
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
object_key: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
class MessageAttachment(Base):
|
||||
__tablename__ = "message_attachments"
|
||||
__table_args__ = (
|
||||
ForeignKeyConstraint(
|
||||
["message_id", "chat_id"],
|
||||
["messages.id", "messages.chat_id"],
|
||||
name="message_attachments_message_chat_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
ForeignKeyConstraint(
|
||||
["attachment_id", "chat_id"],
|
||||
["attachments.id", "attachments.chat_id"],
|
||||
name="message_attachments_attachment_chat_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
UniqueConstraint("message_id", "position", name="message_attachments_position_uq"),
|
||||
Index("message_attachments_attachment_idx", "attachment_id"),
|
||||
)
|
||||
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
|
||||
position: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Find orphan objects older than a safety grace period; deletion requires --apply."""
|
||||
import argparse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import os
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from sqlalchemy import select
|
||||
|
||||
from .db import SessionLocal
|
||||
from .models import Attachment
|
||||
from .settings import _secret
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--apply", action="store_true", help="delete confirmed stale orphans")
|
||||
args = parser.parse_args()
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/")
|
||||
region = os.environ.get("S3_REGION", "us-east-1")
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
try:
|
||||
grace_seconds = int(os.environ.get("ORPHAN_GRACE_SECONDS", "86400"))
|
||||
except ValueError as exc:
|
||||
raise SystemExit("ORPHAN_GRACE_SECONDS must be an integer") from exc
|
||||
if grace_seconds < 3600:
|
||||
raise SystemExit("ORPHAN_GRACE_SECONDS must be at least 3600 seconds")
|
||||
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=region,
|
||||
aws_access_key_id=_secret("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY"),
|
||||
aws_secret_access_key=_secret("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY"),
|
||||
config=Config(connect_timeout=2, read_timeout=5, retries={"total_max_attempts": 1}, s3={"addressing_style": "path"}),
|
||||
)
|
||||
with SessionLocal() as session:
|
||||
known_keys = set(session.scalars(select(Attachment.object_key)).all())
|
||||
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
scanned = stale = deleted = 0
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=bucket, Prefix="originals/"):
|
||||
for obj in page.get("Contents", []):
|
||||
scanned += 1
|
||||
key = obj["Key"]
|
||||
if key in known_keys or obj.get("LastModified") is None or obj["LastModified"] > cutoff:
|
||||
continue
|
||||
stale += 1
|
||||
if args.apply:
|
||||
client.delete_object(Bucket=bucket, Key=key)
|
||||
deleted += 1
|
||||
print(
|
||||
f"orphan_reconciliation mode={'apply' if args.apply else 'dry-run'} "
|
||||
f"grace_seconds={grace_seconds} scanned={scanned} stale={stale} deleted={deleted}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Idempotently create the private bucket and a bucket-scoped API service account."""
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
import boto3
|
||||
from botocore.auth import S3SigV4Auth
|
||||
from botocore.awsrequest import AWSRequest
|
||||
from botocore.config import Config
|
||||
from botocore.credentials import Credentials
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from .settings import _secret
|
||||
|
||||
|
||||
def _admin_request(method: str, path: str, payload: dict | None = None) -> tuple[int, str | None]:
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/")
|
||||
region = os.environ.get("S3_REGION", "us-east-1")
|
||||
access = _secret("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY")
|
||||
secret = _secret("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY")
|
||||
body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") if payload is not None else None
|
||||
url = endpoint + path
|
||||
# RustFS 1.0.5's admin API may reject a signed GET with an empty body
|
||||
# when Content-Length is omitted by urllib. Sign and send the explicit
|
||||
# zero length so the request is unambiguous to its SigV4 verifier.
|
||||
headers = {"Content-Type": "application/json"} if body is not None else {"Content-Length": "0"}
|
||||
signed = AWSRequest(method=method, url=url, data=body, headers=headers)
|
||||
S3SigV4Auth(Credentials(access, secret), "s3", region).add_auth(signed)
|
||||
request = Request(url, data=body, headers=dict(signed.headers), method=method)
|
||||
try:
|
||||
with urlopen(request, timeout=3) as response:
|
||||
response.read(1024 * 1024)
|
||||
return response.status, None
|
||||
except HTTPError as exc:
|
||||
raw = exc.read(1024 * 1024)
|
||||
detail = ""
|
||||
try:
|
||||
document = json.loads(raw)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
document = None
|
||||
if isinstance(document, dict):
|
||||
detail = "; ".join(
|
||||
str(document[key])
|
||||
for key in ("Code", "code", "Message", "message", "error")
|
||||
if document.get(key)
|
||||
)
|
||||
if not detail:
|
||||
detail = raw.decode("utf-8", errors="replace")
|
||||
detail = " ".join(detail.split())[:240]
|
||||
# RustFS can echo request details in an error response. Redact every
|
||||
# credential before including a short diagnostic in the container log.
|
||||
for file_variable, value_variable in (
|
||||
("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY"),
|
||||
("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY"),
|
||||
("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"),
|
||||
("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"),
|
||||
):
|
||||
try:
|
||||
credential = _secret(file_variable, value_variable)
|
||||
except RuntimeError:
|
||||
continue
|
||||
detail = detail.replace(credential, "[redacted]")
|
||||
detail = detail.replace(quote(credential, safe=""), "[redacted]")
|
||||
return exc.code, detail or None
|
||||
|
||||
|
||||
def _failure(action: str, status: int, detail: str | None) -> RuntimeError:
|
||||
suffix = f": {detail}" if detail else ""
|
||||
return RuntimeError(f"RustFS API service-account {action} failed with HTTP {status}{suffix}")
|
||||
|
||||
|
||||
def _create_api_service_account(endpoint: str, region: str, bucket: str) -> None:
|
||||
access_key = _secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID")
|
||||
secret_key = _secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY")
|
||||
policy = {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "BucketLocation",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetBucketLocation"],
|
||||
"Resource": [f"arn:aws:s3:::{bucket}"],
|
||||
},
|
||||
{
|
||||
"Sid": "AttachmentObjectsOnly",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": [f"arn:aws:s3:::{bucket}/*"],
|
||||
},
|
||||
],
|
||||
}
|
||||
encoded_key = quote(access_key, safe="")
|
||||
info_path = f"/rustfs/admin/v3/info-service-account?accessKey={encoded_key}"
|
||||
status, detail = _admin_request("GET", info_path)
|
||||
if status == 404:
|
||||
status, detail = _admin_request(
|
||||
"PUT",
|
||||
"/rustfs/admin/v3/add-service-account",
|
||||
{
|
||||
"accessKey": access_key,
|
||||
"secretKey": secret_key,
|
||||
"name": "messenger-api",
|
||||
"description": "Limited access to the private attachment bucket",
|
||||
"policy": policy,
|
||||
},
|
||||
)
|
||||
if status not in (200, 201, 204):
|
||||
raise _failure("creation", status, detail)
|
||||
elif status != 200:
|
||||
raise _failure("lookup", status, detail)
|
||||
else:
|
||||
update_path = f"/rustfs/admin/v3/update-service-account?accessKey={encoded_key}"
|
||||
status, detail = _admin_request(
|
||||
"POST",
|
||||
update_path,
|
||||
{"policy": policy, "name": "messenger-api", "description": "Limited access to the private attachment bucket"},
|
||||
)
|
||||
if status not in (200, 204):
|
||||
raise _failure("policy update", status, detail)
|
||||
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=region,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=Config(connect_timeout=2, read_timeout=3, retries={"total_max_attempts": 1}, s3={"addressing_style": "path"}),
|
||||
)
|
||||
client.get_bucket_location(Bucket=bucket)
|
||||
try:
|
||||
response = client.list_buckets()
|
||||
except ClientError as exc:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
if code not in {"AccessDenied", "403"}:
|
||||
raise RuntimeError("RustFS API credentials were not explicitly denied ListAllMyBuckets") from None
|
||||
else:
|
||||
# RustFS follows MinIO-compatible filtered listing: ListBuckets may
|
||||
# succeed while returning only buckets this identity can access.
|
||||
# Treat unrelated visible buckets as a leak, not the filtered result.
|
||||
visible = {item.get("Name") for item in response.get("Buckets", [])}
|
||||
if visible - {bucket}:
|
||||
raise RuntimeError("RustFS API credentials can see buckets outside the attachment bucket")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/")
|
||||
region = os.environ.get("S3_REGION", "us-east-1")
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
root_access = _secret("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY")
|
||||
root_secret = _secret("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY")
|
||||
root = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=region,
|
||||
aws_access_key_id=root_access,
|
||||
aws_secret_access_key=root_secret,
|
||||
config=Config(connect_timeout=2, read_timeout=3, retries={"total_max_attempts": 1}, s3={"addressing_style": "path"}),
|
||||
)
|
||||
last_error = None
|
||||
for _ in range(60):
|
||||
try:
|
||||
try:
|
||||
root.head_bucket(Bucket=bucket)
|
||||
except ClientError as exc:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
if code in {"404", "NoSuchBucket", "NotFound"}:
|
||||
root.create_bucket(Bucket=bucket)
|
||||
else:
|
||||
raise
|
||||
break
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError("RustFS did not become ready for bucket bootstrap") from last_error
|
||||
|
||||
anonymous_url = f"{endpoint}/{quote(bucket, safe='')}"
|
||||
try:
|
||||
with urlopen(Request(anonymous_url, method="GET"), timeout=2) as response:
|
||||
if response.status < 400:
|
||||
raise RuntimeError("The attachment bucket allows anonymous access")
|
||||
except HTTPError as exc:
|
||||
if exc.code != 403:
|
||||
raise RuntimeError(f"Anonymous bucket check returned unexpected HTTP {exc.code}") from None
|
||||
except URLError as exc:
|
||||
raise RuntimeError("Could not verify private bucket access") from exc
|
||||
|
||||
_create_api_service_account(endpoint, region, bucket)
|
||||
print("PASS: private attachment bucket and least-privilege API credentials are ready.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,123 @@
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator
|
||||
import uuid
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CreateUser(StrictModel):
|
||||
username: StrictStr = Field(min_length=3, max_length=32, pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||
password: StrictStr = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def display_name_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("display_name must contain a non-whitespace character")
|
||||
return value
|
||||
|
||||
|
||||
class UserOut(BaseModel):
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class CreateChat(StrictModel):
|
||||
title: StrictStr = Field(min_length=1, max_length=100)
|
||||
member_ids: list[StrictStr] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def title_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("title must contain a non-whitespace character")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
@classmethod
|
||||
def members_are_unique_uuids(cls, values: list[str]) -> list[str]:
|
||||
import uuid
|
||||
|
||||
try:
|
||||
normalized = [str(uuid.UUID(value)) for value in values]
|
||||
except (ValueError, AttributeError, TypeError) as exc:
|
||||
raise ValueError("member_ids must contain UUIDs") from exc
|
||||
if len(normalized) != len(set(normalized)):
|
||||
raise ValueError("member_ids must be unique")
|
||||
return normalized
|
||||
|
||||
|
||||
class ChatOut(BaseModel):
|
||||
id: str
|
||||
title: str
|
||||
member_ids: list[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class ChatPage(BaseModel):
|
||||
items: list[ChatOut]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class CreateMessage(StrictModel):
|
||||
text: StrictStr = Field(max_length=4000)
|
||||
attachment_ids: list[StrictStr] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def attachment_ids_are_unique_uuids(cls, values: list[str]) -> list[str]:
|
||||
try:
|
||||
normalized = [str(uuid.UUID(value)) for value in values]
|
||||
except (ValueError, AttributeError, TypeError) as exc:
|
||||
raise ValueError("attachment_ids must contain UUIDs") from exc
|
||||
if len(normalized) != len(set(normalized)):
|
||||
raise ValueError("attachment_ids must be unique")
|
||||
return normalized
|
||||
|
||||
|
||||
class MessageOut(BaseModel):
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: list[str]
|
||||
|
||||
|
||||
class AttachmentOut(BaseModel):
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str = "ready"
|
||||
variants: list[dict] = Field(default_factory=list)
|
||||
error_code: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class MessagePage(BaseModel):
|
||||
items: list[MessageOut]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class SessionOut(BaseModel):
|
||||
token: str
|
||||
token_type: str = "Bearer"
|
||||
expires_at: str
|
||||
user: UserOut
|
||||
|
||||
|
||||
class Health(BaseModel):
|
||||
status: str = "ok"
|
||||
|
||||
|
||||
class ErrorBody(BaseModel):
|
||||
error: dict[str, str]
|
||||
@@ -0,0 +1,67 @@
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _secret(file_variable: str, value_variable: str) -> str:
|
||||
file_path = os.environ.get(file_variable)
|
||||
if file_path:
|
||||
value = Path(file_path).read_text(encoding="utf-8").strip()
|
||||
else:
|
||||
value = os.environ.get(value_variable, "").strip()
|
||||
if not value:
|
||||
raise RuntimeError(f"Required setting {file_variable} is missing")
|
||||
return value
|
||||
|
||||
|
||||
def load_database_url() -> str:
|
||||
return _secret("DATABASE_URL_FILE", "DATABASE_URL")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
cache_url: str
|
||||
session_ttl_seconds: int
|
||||
s3_endpoint: str
|
||||
s3_region: str
|
||||
s3_bucket: str
|
||||
s3_access_key_id: str
|
||||
s3_secret_access_key: str
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
try:
|
||||
ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS must be an integer") from exc
|
||||
if ttl < 1 or ttl > 31_536_000:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS must be between 1 and 31536000")
|
||||
try:
|
||||
max_upload_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
|
||||
max_image_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("MAX_UPLOAD_BYTES and MAX_IMAGE_PIXELS must be integers") from exc
|
||||
if not 1 <= max_upload_bytes <= 10_485_760:
|
||||
raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10485760")
|
||||
if not 1 <= max_image_pixels <= 40_000_000:
|
||||
raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40000000")
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").strip()
|
||||
region = os.environ.get("S3_REGION", "us-east-1").strip()
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip()
|
||||
if not endpoint.startswith("http://") and not endpoint.startswith("https://"):
|
||||
raise RuntimeError("S3_ENDPOINT must be an HTTP(S) URL")
|
||||
if not region or not bucket:
|
||||
raise RuntimeError("S3_REGION and S3_BUCKET are required")
|
||||
return Settings(
|
||||
cache_url=_secret("CACHE_URL_FILE", "CACHE_URL"),
|
||||
session_ttl_seconds=ttl,
|
||||
s3_endpoint=endpoint.rstrip("/"),
|
||||
s3_region=region,
|
||||
s3_bucket=bucket,
|
||||
s3_access_key_id=_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"),
|
||||
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"),
|
||||
max_upload_bytes=max_upload_bytes,
|
||||
max_image_pixels=max_image_pixels,
|
||||
)
|
||||
@@ -0,0 +1,20 @@
|
||||
from botocore.config import Config
|
||||
import boto3
|
||||
|
||||
from .settings import load_settings
|
||||
|
||||
|
||||
settings = load_settings()
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.s3_access_key_id,
|
||||
aws_secret_access_key=settings.s3_secret_access_key,
|
||||
config=Config(
|
||||
connect_timeout=1,
|
||||
read_timeout=2,
|
||||
retries={"total_max_attempts": 1, "mode": "standard"},
|
||||
s3={"addressing_style": "path"},
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade-4 attachment rejection checks; uses only the public HTTP contract."""
|
||||
import uuid
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab 4 grade-4 negative attachment checks")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
base_path = "/api/v1/chats/" + chat["id"] + "/attachments"
|
||||
valid_png = (Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
too_large = b"\x00" * (10 * 1024 * 1024 + 1)
|
||||
oversized = client.call(
|
||||
"POST", base_path + "?filename=large.png", 413,
|
||||
actor=alice, raw=too_large, headers={"Content-Type": "image/png"},
|
||||
)
|
||||
corrupt = client.call(
|
||||
"POST", base_path + "?filename=broken.png", 415,
|
||||
actor=alice, raw=b"not a PNG", headers={"Content-Type": "image/png"},
|
||||
)
|
||||
spoofed = client.call(
|
||||
"POST", base_path + "?filename=spoofed.jpg", 415,
|
||||
actor=alice, raw=valid_png, headers={"Content-Type": "image/jpeg"},
|
||||
)
|
||||
unsafe_name = client.call(
|
||||
"POST", base_path + "?filename=..%2Foutside.png", 400,
|
||||
actor=alice, raw=valid_png, headers={"Content-Type": "image/png"},
|
||||
)
|
||||
|
||||
client.attachment(alice, chat)
|
||||
second_chat = client.call(
|
||||
"POST", "/api/v1/chats", 201, actor=bob,
|
||||
body={"title": "Другой чат", "member_ids": [bob["id"], eve["id"]]}, schema="Chat",
|
||||
)["json"]
|
||||
foreign_attachment, _ = client.attachment(bob, second_chat)
|
||||
foreign = client.call(
|
||||
"POST", "/api/v1/chats/" + chat["id"] + "/messages", 404,
|
||||
actor=alice,
|
||||
body={"text": "чужое вложение", "attachment_ids": [foreign_attachment["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
unknown = client.call(
|
||||
"POST", "/api/v1/chats/" + chat["id"] + "/messages", 404,
|
||||
actor=alice,
|
||||
body={"text": "неизвестное вложение", "attachment_ids": [str(uuid.uuid4())]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
summary = {
|
||||
"oversized_upload": oversized["status"],
|
||||
"corrupt_image": corrupt["status"],
|
||||
"mime_spoof": spoofed["status"],
|
||||
"unsafe_filename": unsafe_name["status"],
|
||||
"foreign_chat_attachment": foreign["status"],
|
||||
"unknown_attachment": unknown["status"],
|
||||
"result": "PASS",
|
||||
}
|
||||
evidence = Path(__file__).resolve().parents[1] / "evidence/grade4-attachments.json"
|
||||
evidence.parent.mkdir(parents=True, exist_ok=True)
|
||||
evidence.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(summary, separators=(",", ":")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove shared message idempotency across concurrent API replicas."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument("--workers", type=int, default=12)
|
||||
parser.add_argument("--requests", type=int, default=24)
|
||||
|
||||
args = arguments("Concurrent duplicate message POST across API replicas", configure)
|
||||
check(args.workers >= 2 and args.requests >= args.workers, "requests must be >= workers >= 2")
|
||||
setup = Client(args.base_url, args.ca_file, timeout=10)
|
||||
alice, bob, _eve, chat = setup.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
body = "parallel-idempotency-" + uuid.uuid4().hex
|
||||
barrier = threading.Barrier(args.workers)
|
||||
|
||||
def send(index):
|
||||
client = Client(args.base_url, args.ca_file, timeout=10)
|
||||
if index < args.workers:
|
||||
barrier.wait(timeout=10)
|
||||
result = client.message(alice, chat, body, key=key)
|
||||
return result["json"], result["headers"]["X-Instance-Id"]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
||||
results = list(pool.map(send, range(args.requests)))
|
||||
messages = [item[0] for item in results]
|
||||
instances = {item[1] for item in results}
|
||||
check(all(message == messages[0] for message in messages), "same key/body returned different messages")
|
||||
check(len(instances) >= 2, "concurrent requests did not reach two API replicas")
|
||||
history = setup.call(
|
||||
"GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100",
|
||||
actor=bob, schema="MessagePage",
|
||||
)["json"]["items"]
|
||||
stored = [item for item in history if item["text"] == body]
|
||||
check(len(stored) == 1, "concurrent duplicate created zero or multiple stored messages")
|
||||
print(json.dumps({
|
||||
"parallel_requests": args.requests,
|
||||
"workers": args.workers,
|
||||
"replicas_observed": len(instances),
|
||||
"unique_message_ids": len({item["id"] for item in messages}),
|
||||
"stored_copies": len(stored),
|
||||
"result": "PASS",
|
||||
}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stop RustFS briefly and confirm the API fails closed, then restore it."""
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", *args],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Bounded S3 outage check; stops and restores only the local RustFS service")
|
||||
client = Client(args.base_url, args.ca_file, timeout=12)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
stopped = False
|
||||
results = {}
|
||||
try:
|
||||
compose("stop", "s3")
|
||||
stopped = True
|
||||
live = client.call("GET", "/health/live", schema="Health")
|
||||
check(live["json"]["status"] == "ok", "liveness failed during S3 outage")
|
||||
results["liveness"] = live["status"]
|
||||
results["readiness_during_outage"] = client.call("GET", "/health/ready", 503)["status"]
|
||||
image = (Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png").read_bytes()
|
||||
upload = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/attachments?filename=s3-outage.png",
|
||||
503,
|
||||
actor=alice,
|
||||
raw=image,
|
||||
headers={"Content-Type": "image/png"},
|
||||
)
|
||||
# urllib response headers are an HTTPMessage, not JSON serializable.
|
||||
# Record only the safe scalar evidence needed for this outage check.
|
||||
results["upload_during_outage"] = {
|
||||
"status": upload["status"],
|
||||
"error_code": upload["json"]["error"]["code"],
|
||||
"elapsed_seconds": round(upload["elapsed"], 3),
|
||||
}
|
||||
finally:
|
||||
if stopped:
|
||||
compose("start", "s3")
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
client.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Failure:
|
||||
if time.monotonic() >= deadline:
|
||||
raise
|
||||
time.sleep(1)
|
||||
print("PASS: RustFS is ready again.")
|
||||
results["readiness_after_restore"] = 200
|
||||
results["result"] = "PASS"
|
||||
evidence = Path(__file__).resolve().parents[1] / "evidence/s3-outage.json"
|
||||
evidence.parent.mkdir(parents=True, exist_ok=True)
|
||||
evidence.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(results, separators=(",", ":")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload ten decoded 3000x3000 PNGs concurrently and sample Docker API memory."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zlib
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def png_chunk(kind: bytes, payload: bytes) -> bytes:
|
||||
return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def large_decode_png(width=3000, height=3000) -> bytes:
|
||||
row = b"\x00" + (b"\x28\x80\xc0" * width)
|
||||
compressor = zlib.compressobj(level=6)
|
||||
compressed = bytearray()
|
||||
for _ in range(height):
|
||||
compressed.extend(compressor.compress(row))
|
||||
compressed.extend(compressor.flush())
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", bytes(compressed)) + png_chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def memory_mib(containers):
|
||||
result = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.ID}}|{{.MemUsage}}", *containers],
|
||||
check=True, capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
samples = {}
|
||||
for line in result.stdout.splitlines():
|
||||
fields = line.split("|", 1)
|
||||
if len(fields) != 2:
|
||||
continue
|
||||
number = fields[1].split("/", 1)[0].strip().split()[0]
|
||||
match = re.fullmatch(r"([0-9.]+)(B|KiB|MiB|GiB)", number)
|
||||
if not match:
|
||||
continue
|
||||
value = float(match.group(1))
|
||||
value_mib = value * {"GiB": 1024, "MiB": 1, "KiB": 1 / 1024, "B": 1 / (1024 * 1024)}[match.group(2)]
|
||||
samples[fields[0]] = value_mib
|
||||
return samples
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--ca-file", default=None)
|
||||
parser.add_argument("--concurrency", type=int, default=10)
|
||||
parser.add_argument("--output", default="evidence/upload-memory.json")
|
||||
args = parser.parse_args()
|
||||
check(args.concurrency >= 10, "grade 4 memory run requires at least 10 concurrent clients")
|
||||
client = Client(args.base_url, args.ca_file, timeout=90)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
image = large_decode_png()
|
||||
containers = subprocess.check_output(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], text=True, timeout=10,
|
||||
).split()
|
||||
check(len(containers) >= 2, "start at least two API replicas behind the load balancer")
|
||||
before = memory_mib(containers)
|
||||
peak = dict(before)
|
||||
stop = threading.Event()
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
for container, value in memory_mib(containers).items():
|
||||
peak[container] = max(peak.get(container, 0), value)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
pass
|
||||
stop.wait(0.25)
|
||||
|
||||
monitor = threading.Thread(target=sample, daemon=True)
|
||||
monitor.start()
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=memory-probe.png"
|
||||
|
||||
def upload(_index):
|
||||
request = urllib.request.Request(
|
||||
client.base + path,
|
||||
data=image,
|
||||
headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with client.opener.open(request, timeout=90) as response:
|
||||
response.read(1024 * 1024)
|
||||
check(response.status == 201, "upload did not return 201")
|
||||
return response.headers.get("X-Instance-Id", "")
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise AssertionError(f"upload returned HTTP {exc.code}") from None
|
||||
|
||||
started = time.monotonic()
|
||||
successes = []
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
futures = [pool.submit(upload, i) for i in range(args.concurrency)]
|
||||
for future in as_completed(futures):
|
||||
successes.append(future.result())
|
||||
finally:
|
||||
stop.set()
|
||||
monitor.join(timeout=3)
|
||||
report = {
|
||||
"concurrent_clients": args.concurrency,
|
||||
"successful_uploads": len(successes),
|
||||
"decoded_dimensions": [3000, 3000],
|
||||
"input_bytes": len(image),
|
||||
"container_memory_mib_before": before,
|
||||
"container_memory_mib_peak": peak,
|
||||
"elapsed_seconds": round(time.monotonic() - started, 3),
|
||||
"observed_instances": sorted({item for item in successes if item}),
|
||||
}
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, separators=(",", ":")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user