ЛР 4 — БВТ2302 — Бадрадин Али — на 4 #138

Closed
Badradine wants to merge 1 commits from Badradine/mtusi2026lab4:solution/БВТ2302-Бадрадин-Али into main
36 changed files with 2740 additions and 73 deletions
+7 -20
View File
@@ -1,20 +1,7 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# Values substituted by compose.yaml. Credentials stay in local files under secrets/.
APP_PORT=8080
SESSION_TTL_SECONDS=3600
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
ORPHAN_GRACE_SECONDS=86400
+27
View File
@@ -0,0 +1,27 @@
FROM python:3.12.6-slim-bookworm
ARG BUILD_REVISION=local
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONUTF8=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_BUILD_REVISION=${BUILD_REVISION}
WORKDIR /app
COPY requirements.txt ./
RUN pip install --no-cache-dir --requirement requirements.txt
COPY src/ ./src/
COPY migrations/ ./migrations/
RUN useradd --create-home --uid 10001 appuser
USER appuser
EXPOSE 8080
HEALTHCHECK --interval=3s --timeout=2s --start-period=10s --retries=3 \
CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)"
CMD ["uvicorn", "src.app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1", "--timeout-graceful-shutdown", "5"]
+45 -30
View File
@@ -1,30 +1,45 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
check:
$(PYTHON) scripts/check_contract.py
validate:
$(PYTHON) scripts/validate_openapi.py
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover test-idempotency test-grade4 test-s3-outage memory-uploads reconcile-s3
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
check:
$(PYTHON) scripts/check_contract.py
validate:
$(PYTHON) scripts/validate_openapi.py
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency:
PYTHONPATH=tests $(PYTHON) student-tests/idempotency_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-grade4:
PYTHONPATH=tests $(PYTHON) student-tests/grade4_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-s3-outage:
PYTHONPATH=tests $(PYTHON) student-tests/s3_outage.py --base-url "$(BASE_URL)" $(TLS_ARGS)
memory-uploads:
PYTHONPATH=tests $(PYTHON) student-tests/upload_memory.py --base-url "$(BASE_URL)" $(TLS_ARGS)
reconcile-s3:
docker compose --env-file .env --profile maintenance run --rm s3-maintenance
+91 -23
View File
@@ -1,37 +1,105 @@
# Отчёт — лабораторная 4
# Отчёт по лабораторной работе 4 — вложения в S3
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор / группа: Бадрадин Али / БВТ2302
- Целевая оценка: 4
- Трек 5: не заявлен
- Репозиторий: `Бадрадин Али/mtusi2026lab4`
- Commit / PR: заполнить после commit и создания PR
- Текущее состояние: реализация и перечисленные ниже проверки выполнены локально в Docker Desktop/WSL; результаты подтверждены выводом команд и файлами в `evidence/`. Commit и PR ещё нужно оформить.
## Запуск из чистого clone
## Как запустить из чистого клона
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Git, Docker Desktop с запущенным Docker Engine, Python 3.12 и PowerShell. Команды приложения запускаются из корня репозитория.
```powershell
git clone https://gitea.privcloud.tech/Badradine/mtusi2026lab4.git
Set-Location mtusi2026lab4
Copy-Item .env.example .env
.\scripts\create-secrets.ps1
python -m venv .venv
.\.venv\Scripts\Activate.ps1
python -m pip install -r requirements-dev.txt
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2
docker compose --env-file .env ps
```
Дождаться, пока обе API-реплики, PostgreSQL, Redis, RustFS и балансировщик будут готовы. Затем выполнить:
```powershell
python scripts/check_contract.py
python scripts/validate_openapi.py
python tests/smoke.py --base-url http://localhost:8080
python student-tests/idempotency_concurrency.py --base-url http://localhost:8080
python student-tests/grade4_attachments.py --base-url http://localhost:8080
python student-tests/s3_outage.py --base-url http://localhost:8080
python student-tests/upload_memory.py --base-url http://localhost:8080 --concurrency 10
docker compose --env-file .env --profile maintenance run --rm s3-maintenance
```
`upload_memory.py` создаёт сводку в `evidence/upload-memory.json`. Для проверки persistence/failover требуется WSL или Git Bash, так как hooks — shell-скрипты. В WSL из корня клона:
```sh
chmod +x scripts/restart.sh scripts/stop-one.sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080
bash scripts/recover-api.sh
```
Фактические результаты текущего запуска записаны в таблице ниже. Сводка persistence/failover основана на выводе WSL, предоставленном автором; остальные JSON-файлы находятся в `evidence/`. Не сохранять пароли, bearer tokens или содержимое файлов-секретов.
Для обычной остановки без удаления данных: `docker compose --env-file .env stop`. Команда `docker compose --env-file .env down -v` намеренно удаляет учебные volumes PostgreSQL, Redis и RustFS вместе с их данными; применять только если тестовый стенд нужно полностью сбросить.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
Клиент → Nginx → API × 2 → PostgreSQL (пользователи, чаты, сообщения, метаданные)
→ Redis (сессии)
→ приватный RustFS/S3 bucket (байты изображений)
```
Сохранены возможности Lab 3: PostgreSQL, Redis, bearer-сессии, Argon2id, миграции, две API-реплики, Nginx и идемпотентность POST сообщений. Добавлены таблицы `attachments` и `message_attachments`, синхронные upload/download PNG/JPEG и связь вложений с чатом и сообщением. Составной внешний ключ в БД не позволяет связать сообщение с вложением из другого чата.
Байты записываются в S3-совместимый RustFS, а PostgreSQL хранит ключ, MIME, имя для отображения, размер, SHA-256 и статус. Ключ объекта создаёт сервер; имя файла никогда не становится путём или S3-ключом. API возвращает `201 ready` только после записи объекта и commit метаданных. При ошибке после S3 PUT выполняется удаление объекта как компенсация. Если процесс аварийно остановится между двумя операциями и компенсация не выполнится, `s3-maintenance` обнаруживает старые объекты без строки БД. По умолчанию это только dry-run; автоматическое удаление возможно только с `--apply` и grace period не меньше одного часа. Миграция `004_attachment_schema_compat` без удаления данных приводит ранее созданную схему `003_attachments` к текущим полям metadata и составной связи с чатом.
Upload читает тело потоково в spooled temporary file: до 1 MiB хранится в памяти, остальное уходит во временный файл контейнера, размер ограничен 10 MiB. Семафор ограничивает параллельные декодирования каждого API-процесса одним запросом; временный `/tmp` имеет лимит 64 MiB. До полной декомпрессии проверяются PNG/JPEG signature/формат, совпадение MIME, размер изображения и предел 40 млн пикселей. S3 timeout ограничен, SDK настроен на одну попытку.
Bootstrap создаёт bucket, если его ещё нет, задаёт отдельную service-account key для API с правами только `GetBucketLocation`, `GetObject`, `PutObject` и `DeleteObject` в этом bucket, проверяет приватность анонимным чтением и отсутствие доступа к посторонним bucket. RustFS может вернуть успешный `ListBuckets` с отфильтрованным перечнем разрешённых bucket; bootstrap принимает такой ответ, только если в нём нет bucket вне хранилища вложений. Root-секреты доступны только bootstrap и вручную запускаемому maintenance-сервису; API получает отдельные ограниченные ключи. Перед запуском RustFS одноразовый `s3-init` выставляет владельца тома `s3_data` в UID/GID `10001:10001`, которые используются непривилегированным процессом RustFS. Порты S3 и Console не опубликованы на хосте, встроенная административная Console выключена. Образ RustFS зафиксирован на `rustfs/rustfs:1.0.5`; данные лежат в named volume `s3_data`. Это локальная одноузловая учебная конфигурация, не отказоустойчивая production-система.
### Соответствие конфигурации
`.env.example` содержит только используемые Compose-переменные: `APP_PORT`, `SESSION_TTL_SECONDS`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` и `ORPHAN_GRACE_SECONDS`. Реальные URI БД/Redis и S3 credentials монтируются из локальных файлов в `secrets/`. Файл `.env` и каталог `secrets/` исключены из Git. `scripts/create-secrets.ps1` создаёт отсутствующие пары, не печатает значения и не меняет уже созданные полные пары.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
Таблица отражает фактические результаты, полученные автором на локальном стенде. `check_contract.py` и `validate_openapi.py` проверяют структуру/схему, но сами по себе не запускают приложение.
| Критерий | Команда | Наблюдаемый результат / доказательство | Состояние |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Структура и синтаксис Python | `python scripts/check_contract.py` | `PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.` | PASS |
| Полная OpenAPI-валидация | `python scripts/validate_openapi.py` | `PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.` | PASS |
| Публичный контракт, доступы и checksum оригинала | `python tests/smoke.py --base-url http://localhost:8080` | Smoke-сценарий Lab 4 завершился `PASS` после применения миграции совместимости 004 | PASS |
| Сохранение Lab 3 и идемпотентность на репликах | `python student-tests/idempotency_concurrency.py --base-url http://localhost:8080` | 24 параллельных запроса, 12 workers, обнаружены 2 реплики, одна сохранённая копия | PASS |
| Слишком большой файл, повреждённый PNG, MIME spoof, опасное имя | `python student-tests/grade4_attachments.py --base-url http://localhost:8080` | Коды 413, 415, 415 и 400; сводка `evidence/grade4-attachments.json` | PASS |
| Чужой и неизвестный attachment ID | Та же команда grade-4 негативных проверок | Для обоих случаев возвращён 404; сводка `evidence/grade4-attachments.json` | PASS |
| Недоступность S3 и восстановление | `python student-tests/s3_outage.py --base-url http://localhost:8080` | Liveness 200; readiness и upload во время сбоя 503; после восстановления readiness 200. `evidence/s3-outage.json` | PASS |
| Ограниченная потоковая память при ≥10 upload | `python student-tests/upload_memory.py --base-url http://localhost:8080 --concurrency 10` | 10/10 upload успешны, обе реплики наблюдались; RAM до 134.8/143.0 MiB, пик 150.2/151.0 MiB. `evidence/upload-memory.json` | PASS |
| Идемпотентный bootstrap, приватный bucket, ограниченные права | `docker compose --env-file .env up -d --build --scale api=2` | `s3-bootstrap` завершился успешно: приватный bucket и ограниченные API credentials готовы; API и LB здоровы | PASS |
| Persistence всех хранилищ и прежней сессии/файла | `make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080` | Существующие подтверждённые данные и сессия сохранились после restart. См. `evidence/resilience-results.md` | PASS |
| Отказ одной API-реплики и восстановление через LB | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080` | 2 реплики наблюдались до отказа; 84 успешные записи, 0 неудачных попыток, первое восстановление за 1.578 с. Затем `recover-api.sh`: 2 реплики здоровы через LB. См. `evidence/resilience-results.md` | PASS |
| Dry-run поиска S3 orphan | `docker compose --env-file .env --profile maintenance run --rm s3-maintenance` | `mode=dry-run`, scanned=13, stale=0, deleted=0 | PASS |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Ключи, секреты, содержимое secret-файлов и bearer tokens не включать в отчёт. Не отмечать checklist PR до подтверждения пунктов выводом.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
В этой версии не заявлен трек 5. Перед commit/PR нужно проверить `docker compose ps`, просмотреть `git diff`/`git status` и убедиться, что секреты не попали в Git. Вывод resilience сообщает `Hook scope needs operator evidence`; результаты подтверждают поведение сценариев теста, а не внешнее подтверждение администраторских действий. RustFS в Compose — одиночный локальный узел с одним volume, поэтому его отказоустойчивость и сохранность диска не эквивалентны кластерному object storage или backup.
## Источники и инструменты
- RustFS закреплённый release `1.0.5`: https://github.com/RustFS/RustFS/releases/tag/1.0.5
- RustFS service accounts и ограничивающие policy: https://docs.rustfs.com/en/security-compliance/iam/sts и https://docs.rustfs.com/en/security-compliance/iam/policies
- RustFS readiness endpoint: https://docs.rustfs.com/en/operations/status-check
- boto3 `1.44.1`: https://pypi.org/project/boto3/1.44.1/
- Pillow `13.0.0`: https://pypi.org/project/pillow/13.0.0/
- Для реализации и ревью использован Codex; проверки запускал автор на локальном стенде. Приведённые результаты относятся к этому запуску и не заменяют повторную проверку при изменении кода или окружения.
+292
View File
@@ -0,0 +1,292 @@
name: mtusi-messenger-lab4
services:
db:
image: postgres:16.4-alpine
restart: unless-stopped
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
volumes:
- postgres_data:/var/lib/postgresql/data
networks:
- backend
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 3s
timeout: 3s
retries: 20
start_period: 5s
deploy:
resources:
limits:
cpus: "1.00"
memory: 1G
redis:
image: redis:7.4.1-alpine
restart: unless-stopped
command:
- sh
- -ec
- exec redis-server --dir /data --appendonly yes --appendfsync everysec --save 60 1 --requirepass "$$(cat /run/secrets/redis_password)"
secrets:
- redis_password
volumes:
- redis_data:/data
networks:
- backend
healthcheck:
test:
- CMD-SHELL
- 'REDISCLI_AUTH="$$(cat /run/secrets/redis_password)" redis-cli ping | grep -q PONG'
interval: 3s
timeout: 3s
retries: 20
start_period: 5s
deploy:
resources:
limits:
cpus: "0.25"
memory: 256M
s3-init:
image: alpine:3.24.1
user: "0:0"
command: ["sh", "-ec", "chown -R 10001:10001 /data && chmod 0750 /data"]
volumes:
- s3_data:/data
restart: "no"
s3:
image: rustfs/rustfs:1.0.5
restart: unless-stopped
environment:
RUSTFS_VOLUMES: /data
RUSTFS_ADDRESS: 0.0.0.0:9000
RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001
RUSTFS_CONSOLE_ENABLE: "false"
RUSTFS_OBS_LOGGER_LEVEL: warn
RUSTFS_CHECK_UPDATE: "false"
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key_id
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_secret_access_key
secrets:
- s3_root_access_key_id
- s3_root_secret_access_key
volumes:
- s3_data:/data
networks:
- backend
depends_on:
s3-init:
condition: service_completed_successfully
healthcheck:
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready >/dev/null"]
interval: 3s
timeout: 2s
retries: 20
start_period: 10s
deploy:
resources:
limits:
cpus: "1.00"
memory: 1G
migrate:
image: mtusi-messenger-lab4-api:local
build:
context: .
dockerfile: Dockerfile
args:
BUILD_REVISION: ${BUILD_REVISION:-local}
command: ["python", "-m", "src.app.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
networks:
- backend
depends_on:
db:
condition: service_healthy
restart: "no"
deploy:
resources:
limits:
cpus: "0.25"
memory: 256M
s3-bootstrap:
image: mtusi-messenger-lab4-api:local
build:
context: .
dockerfile: Dockerfile
args:
BUILD_REVISION: ${BUILD_REVISION:-local}
command: ["python", "-m", "src.app.s3_bootstrap"]
environment:
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ROOT_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key_id
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets:
- s3_root_access_key_id
- s3_root_secret_access_key
- s3_access_key_id
- s3_secret_access_key
networks:
- backend
depends_on:
s3:
condition: service_healthy
restart: "no"
deploy:
resources:
limits:
cpus: "0.25"
memory: 256M
api:
image: mtusi-messenger-lab4-api:local
build:
context: .
dockerfile: Dockerfile
args:
BUILD_REVISION: ${BUILD_REVISION:-local}
restart: unless-stopped
stop_grace_period: 8s
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
secrets:
- database_url
- cache_url
- s3_access_key_id
- s3_secret_access_key
networks:
- backend
- edge
depends_on:
db:
condition: service_healthy
redis:
condition: service_healthy
s3:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2)"]
interval: 3s
timeout: 2s
retries: 3
start_period: 10s
tmpfs:
- /tmp:size=64m,noexec,nosuid
deploy:
resources:
limits:
cpus: "0.50"
memory: 512M
lb:
image: nginx:1.28.0-alpine
restart: unless-stopped
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
volumes:
- ./infra/nginx.conf:/etc/nginx/conf.d/default.conf:ro
networks:
- edge
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
interval: 3s
timeout: 2s
retries: 5
start_period: 5s
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
s3-maintenance:
image: mtusi-messenger-lab4-api:local
build:
context: .
dockerfile: Dockerfile
args:
BUILD_REVISION: ${BUILD_REVISION:-local}
profiles: ["maintenance"]
command: ["python", "-m", "src.app.reconcile_s3_orphans"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ROOT_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key_id
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-86400}
secrets:
- database_url
- s3_root_access_key_id
- s3_root_secret_access_key
networks:
- backend
depends_on:
db:
condition: service_healthy
s3:
condition: service_healthy
migrate:
condition: service_completed_successfully
restart: "no"
secrets:
postgres_password:
file: ./secrets/postgres_password
redis_password:
file: ./secrets/redis_password
s3_root_access_key_id:
file: ./secrets/s3_root_access_key_id
s3_root_secret_access_key:
file: ./secrets/s3_root_secret_access_key
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
volumes:
postgres_data:
redis_data:
s3_data:
networks:
backend:
internal: true
edge:
+9
View File
@@ -0,0 +1,9 @@
{
"oversized_upload": 413,
"corrupt_image": 415,
"mime_spoof": 415,
"unsafe_filename": 400,
"foreign_chat_attachment": 404,
"unknown_attachment": 404,
"result": "PASS"
}
+37
View File
@@ -0,0 +1,37 @@
# Результаты persistence и failover
Сводка составлена по выводу WSL, предоставленному автором. Hooks запущены через цели Makefile репозитория.
## Persistence: перезапуск хранилищ
Commande :
```sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080
```
Результат: `PASS: persistence preserved confirmed data and existing session.`
## Failover одной API-реплики
Commande :
```sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080
```
Résumé JSON affiché par le test :
```json
{"observed_instances_before":2,"successful_writes":84,"failed_attempts":0,"first_recovery_seconds":1.578}
```
Результат: `PASS: failover preserved confirmed data and existing session.`
## Восстановление после теста
Commande : `bash scripts/recover-api.sh`
Результат: `PASS: two API replicas are healthy through the load balancer.`
Тест также вывел `Hook scope needs operator evidence`. Эта сводка фиксирует результат тестового сценария, сообщённый автором; она не является независимым подтверждением состояния Docker.
+11
View File
@@ -0,0 +1,11 @@
{
"liveness": 200,
"readiness_during_outage": 503,
"upload_during_outage": {
"status": 503,
"error_code": "unavailable",
"elapsed_seconds": 9.187
},
"readiness_after_restore": 200,
"result": "PASS"
}
+1
View File
@@ -0,0 +1 @@
2026-11-13T22:46:05Z db93123e60d5
+22
View File
@@ -0,0 +1,22 @@
{
"concurrent_clients": 10,
"successful_uploads": 10,
"decoded_dimensions": [
3000,
3000
],
"input_bytes": 35063,
"container_memory_mib_before": {
"db93123e60d5": 134.8,
"bad2d3dd6806": 143.0
},
"container_memory_mib_peak": {
"db93123e60d5": 150.2,
"bad2d3dd6806": 151.0
},
"elapsed_seconds": 3.829,
"observed_instances": [
"bad2d3dd6806",
"db93123e60d5"
]
}
+32
View File
@@ -0,0 +1,32 @@
upstream messenger_api {
zone messenger_api 64k;
resolver 127.0.0.11 valid=1s ipv6=off;
server api:8080 resolve max_fails=1 fail_timeout=2s;
keepalive 16;
}
server {
listen 8080;
server_name _;
client_max_body_size 11m;
client_body_buffer_size 64k;
client_body_timeout 15s;
proxy_connect_timeout 1s;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
location / {
proxy_pass http://messenger_api;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 2s;
proxy_request_buffering off;
proxy_buffering off;
}
}
+36
View File
@@ -0,0 +1,36 @@
CREATE TABLE users (
id UUID PRIMARY KEY,
username VARCHAR(32) NOT NULL UNIQUE
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name VARCHAR(100) NOT NULL
CHECK (char_length(display_name) BETWEEN 1 AND 100 AND btrim(display_name) <> ''),
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE chats (
id UUID PRIMARY KEY,
title VARCHAR(100) NOT NULL
CHECK (char_length(title) BETWEEN 1 AND 100 AND btrim(title) <> ''),
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_id_idx ON chat_members(user_id, chat_id);
CREATE TABLE messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL,
sender_id UUID NOT NULL,
text VARCHAR(4000) NOT NULL
CHECK (char_length(text) BETWEEN 1 AND 4000 AND btrim(text) <> ''),
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT messages_sender_is_chat_member_fk
FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members(chat_id, user_id) ON DELETE CASCADE
);
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
+16
View File
@@ -0,0 +1,16 @@
CREATE TABLE idempotency_records (
id UUID PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method VARCHAR(8) NOT NULL,
path VARCHAR(200) NOT NULL,
idempotency_key VARCHAR(128) NOT NULL,
request_hash CHAR(64) NOT NULL,
response_body JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMPTZ NOT NULL,
CONSTRAINT idempotency_records_scope_key_uq
UNIQUE (user_id, method, path, idempotency_key),
CHECK (method = 'POST'),
CHECK (char_length(idempotency_key) BETWEEN 1 AND 128)
);
CREATE INDEX idempotency_records_expires_at_idx ON idempotency_records(expires_at);
+48
View File
@@ -0,0 +1,48 @@
-- Lab 4: file bytes remain in S3; PostgreSQL owns metadata and chat relations.
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_check;
ALTER TABLE messages
ADD CONSTRAINT messages_text_length_ck CHECK (char_length(text) BETWEEN 0 AND 4000);
ALTER TABLE messages
ADD CONSTRAINT messages_id_chat_id_uq UNIQUE (id, chat_id);
CREATE TABLE attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL,
owner_id UUID NOT NULL,
filename VARCHAR(255) NOT NULL,
content_type VARCHAR(32) NOT NULL,
size_bytes INTEGER NOT NULL,
sha256 CHAR(64) NOT NULL,
status VARCHAR(16) NOT NULL DEFAULT 'ready',
variants JSONB NOT NULL DEFAULT '[]'::jsonb,
error_code VARCHAR(64),
object_key TEXT NOT NULL UNIQUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT attachments_owner_is_chat_member_fk
FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members(chat_id, user_id) ON DELETE CASCADE,
CONSTRAINT attachments_id_chat_id_uq UNIQUE (id, chat_id),
CONSTRAINT attachments_filename_length_ck CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_content_type_ck CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size_bytes_ck CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256_ck CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_status_ck CHECK (status = 'ready')
);
CREATE INDEX attachments_chat_created_idx ON attachments(chat_id, created_at, id);
CREATE TABLE message_attachments (
message_id UUID NOT NULL,
attachment_id UUID NOT NULL,
chat_id UUID NOT NULL,
position SMALLINT NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_message_chat_fk
FOREIGN KEY (message_id, chat_id)
REFERENCES messages(id, chat_id) ON DELETE CASCADE,
CONSTRAINT message_attachments_attachment_chat_fk
FOREIGN KEY (attachment_id, chat_id)
REFERENCES attachments(id, chat_id) ON DELETE CASCADE,
CONSTRAINT message_attachments_position_uq UNIQUE (message_id, position)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments(attachment_id);
@@ -0,0 +1,30 @@
-- Bring databases that already recorded an earlier 003_attachments layout
-- forward without dropping attachment metadata or stored object references.
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS variants JSONB NOT NULL DEFAULT '[]'::jsonb;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code VARCHAR(64);
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS width INTEGER;
ALTER TABLE attachments ADD COLUMN IF NOT EXISTS height INTEGER;
ALTER TABLE attachments ALTER COLUMN width DROP NOT NULL;
ALTER TABLE attachments ALTER COLUMN height DROP NOT NULL;
ALTER TABLE message_attachments ADD COLUMN IF NOT EXISTS chat_id UUID;
UPDATE message_attachments AS link
SET chat_id = message.chat_id
FROM messages AS message
WHERE message.id = link.message_id
AND (link.chat_id IS NULL OR link.chat_id <> message.chat_id);
ALTER TABLE message_attachments ALTER COLUMN chat_id SET NOT NULL;
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_id_chat_id_uq;
ALTER TABLE messages ADD CONSTRAINT messages_id_chat_id_uq UNIQUE (id, chat_id);
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_id_chat_id_uq;
ALTER TABLE attachments ADD CONSTRAINT attachments_id_chat_id_uq UNIQUE (id, chat_id);
ALTER TABLE message_attachments DROP CONSTRAINT IF EXISTS message_attachments_message_chat_fk;
ALTER TABLE message_attachments
ADD CONSTRAINT message_attachments_message_chat_fk
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE;
ALTER TABLE message_attachments DROP CONSTRAINT IF EXISTS message_attachments_attachment_chat_fk;
ALTER TABLE message_attachments
ADD CONSTRAINT message_attachments_attachment_chat_fk
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) ON DELETE CASCADE;
+9
View File
@@ -0,0 +1,9 @@
fastapi==0.115.0
uvicorn[standard]==0.30.6
pydantic==2.9.2
SQLAlchemy==2.0.36
psycopg[binary]==3.2.3
redis==5.2.0
argon2-cffi==23.1.0
boto3==1.44.1
Pillow==13.0.0
+57
View File
@@ -0,0 +1,57 @@
$ErrorActionPreference = 'Stop'
$repo = Split-Path -Parent $PSScriptRoot
$secretDir = Join-Path $repo 'secrets'
New-Item -ItemType Directory -Path $secretDir -Force | Out-Null
function New-Secret {
$bytes = New-Object byte[] 32
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
try { $rng.GetBytes($bytes) } finally { $rng.Dispose() }
return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')
}
function New-AccessKey {
$alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
$bytes = New-Object byte[] 20
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
try { $rng.GetBytes($bytes) } finally { $rng.Dispose() }
$value = -join ($bytes | ForEach-Object { $alphabet[($_ % $alphabet.Length)] })
return $value
}
function Read-Secret([string]$name) {
$path = Join-Path $secretDir $name
if (-not (Test-Path -LiteralPath $path)) { return $null }
return [System.IO.File]::ReadAllText($path).Trim()
}
function Write-Secret([string]$name, [string]$value) {
$path = Join-Path $secretDir $name
[System.IO.File]::WriteAllText($path, $value, [System.Text.UTF8Encoding]::new($false))
}
function Ensure-Pair([string]$first, [string]$second, [scriptblock]$newFirst, [scriptblock]$newSecond) {
$left = Read-Secret $first
$right = Read-Secret $second
if (($null -eq $left) -ne ($null -eq $right)) {
throw "Only one file of credential pair '$first'/'$second' exists. Resolve it manually; credentials are never rotated automatically."
}
if ($null -eq $left) {
$left = & $newFirst
$right = & $newSecond $left
Write-Secret $first $left
Write-Secret $second $right
}
return @($left, $right)
}
$pg = Ensure-Pair 'postgres_password' 'database_url' { New-Secret } {
param($password)
"postgresql+psycopg://messenger:$password@db:5432/messenger"
}
$redis = Ensure-Pair 'redis_password' 'cache_url' { New-Secret } {
param($password)
"redis://:$password@redis:6379/0"
}
$s3root = Ensure-Pair 's3_root_access_key_id' 's3_root_secret_access_key' { New-AccessKey } { New-Secret }
$s3api = Ensure-Pair 's3_access_key_id' 's3_secret_access_key' { New-AccessKey } { New-Secret }
Write-Output 'Local Postgres, Redis, RustFS root and limited API credentials are ready under secrets/. Existing complete pairs were left unchanged; values were not displayed.'
+21
View File
@@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose --env-file .env up -d --scale api=2 api lb >/dev/null
deadline=$((SECONDS + 60))
while (( SECONDS < deadline )); do
healthy=0
for container_id in $(docker compose --env-file .env ps -q api); do
state=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id" 2>/dev/null || true)
[[ "$state" == healthy ]] && healthy=$((healthy + 1))
done
if (( healthy >= 2 )) && curl --fail --silent http://localhost:8080/health/ready >/dev/null; then
printf 'PASS: two API replicas are healthy through the load balancer.\n'
exit 0
fi
sleep 1
done
printf 'API replicas did not recover within 60 seconds.\n' >&2
docker compose --env-file .env ps
exit 1
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env sh
set -eu
cd "$(dirname "$0")/.."
# Stop the entry points before every state store; named volumes are left intact.
docker compose stop lb api redis db s3
docker compose start db redis s3
attempt=0
until docker compose exec -T db pg_isready -U messenger -d messenger >/dev/null 2>&1 \
&& docker compose exec -T redis sh -ec 'REDISCLI_AUTH="$(cat /run/secrets/redis_password)" redis-cli ping | grep -q PONG' >/dev/null 2>&1 \
&& docker compose exec -T s3 curl -fsS http://127.0.0.1:9000/health/ready >/dev/null 2>&1; do
attempt=$((attempt + 1))
if [ "$attempt" -ge 60 ]; then
echo 'Database/cache did not become ready within 60 seconds.' >&2
exit 1
fi
sleep 1
done
docker compose --env-file .env up -d migrate s3-bootstrap
docker compose --env-file .env up -d --scale api=2 api lb
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
for container_id in $(docker compose ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
if [[ "$hostname" == "$TARGET_INSTANCE_ID" ]]; then
mkdir -p evidence
printf '%s %s\n' "$(date -u +%FT%TZ)" "$TARGET_INSTANCE_ID" >> evidence/stopped-replicas.txt
docker stop --time 6 "$container_id" >/dev/null
exit 0
fi
done
printf 'No running API container matches TARGET_INSTANCE_ID.\n' >&2
exit 1
+1
View File
@@ -0,0 +1 @@
"""MTUSI Lab 2 API."""
+19
View File
@@ -0,0 +1,19 @@
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from .settings import load_database_url
engine = create_engine(
load_database_url(),
pool_size=5,
max_overflow=5,
pool_timeout=3,
pool_recycle=1800,
pool_pre_ping=True,
connect_args={
"connect_timeout": 3,
"options": "-c statement_timeout=5000",
},
)
SessionLocal = sessionmaker(bind=engine, expire_on_commit=False)
+23
View File
@@ -0,0 +1,23 @@
from fastapi import Request
from fastapi.responses import JSONResponse
class ApiError(Exception):
def __init__(self, code: str, message: str, status: int, challenge: str | None = None):
super().__init__(message)
self.code = code
self.message = message
self.status = status
self.challenge = challenge
def error_response(request: Request, code: str, message: str, status: int, challenge: str | None = None):
request_id = getattr(request.state, "request_id", "unknown")
headers = {"X-Request-Id": request_id, "Cache-Control": "no-store"}
if challenge:
headers["WWW-Authenticate"] = challenge
return JSONResponse(
status_code=status,
headers=headers,
content={"error": {"code": code, "message": message, "request_id": request_id}},
)
+71
View File
@@ -0,0 +1,71 @@
"""Create 10,000 disposable rows and record the indexed keyset-page plan."""
import json
import uuid
from sqlalchemy import text
from .db import engine
def run() -> None:
user_id = uuid.uuid4()
chat_id = uuid.uuid4()
plan_json = None
with engine.connect() as connection:
transaction = connection.begin()
try:
connection.execute(
text("INSERT INTO users(id, username, display_name, password_hash) "
"VALUES (:id, :username, 'Synthetic plan fixture', 'not-a-login-hash')"),
{"id": user_id, "username": "plan_" + uuid.uuid4().hex[:20]},
)
connection.execute(
text("INSERT INTO chats(id, title) VALUES (:id, 'Synthetic plan fixture')"),
{"id": chat_id},
)
connection.execute(
text("INSERT INTO chat_members(chat_id, user_id) VALUES (:chat_id, :user_id)"),
{"chat_id": chat_id, "user_id": user_id},
)
connection.execute(
text("""
INSERT INTO messages(id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), :chat_id, :sender_id,
'synthetic message ' || n::text,
TIMESTAMPTZ '2025-01-01 00:00:00+00' + n * INTERVAL '1 microsecond'
FROM generate_series(1, 10000) AS n
"""),
{"chat_id": chat_id, "sender_id": user_id},
)
connection.exec_driver_sql("ANALYZE messages")
raw = connection.execute(
text("""
EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = :chat_id
AND (created_at, id) > (TIMESTAMPTZ '2025-01-01 00:00:00+00',
'00000000-0000-0000-0000-000000000000'::uuid)
ORDER BY created_at, id
LIMIT 50
"""),
{"chat_id": chat_id},
).scalar_one()
plan_json = raw if isinstance(raw, list) else json.loads(raw)
finally:
transaction.rollback()
document = plan_json[0]
def contains_history_index(node: dict) -> bool:
if node.get("Index Name") == "messages_history_idx":
return True
return any(contains_history_index(child) for child in node.get("Plans", []))
if not contains_history_index(document["Plan"]):
raise SystemExit("Planner did not use messages_history_idx; inspect and revise the query/index.")
print(json.dumps(document, indent=2, sort_keys=True))
if __name__ == "__main__":
run()
+818
View File
@@ -0,0 +1,818 @@
import base64
import binascii
import asyncio
from datetime import datetime, timedelta, timezone
from email.message import Message as EmailMessage
import hashlib
import json
import logging
import re
import secrets
import socket
import tempfile
import unicodedata
import uuid
from argon2 import PasswordHasher, Type
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
from fastapi import FastAPI, Header, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, StreamingResponse
from PIL import Image, UnidentifiedImageError
from botocore.exceptions import BotoCoreError, ClientError
from redis import Redis
from redis.exceptions import RedisError
from sqlalchemy import and_, delete, func, or_, select, text, tuple_, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.exc import DBAPIError, OperationalError, TimeoutError as SQLAlchemyTimeoutError
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException as StarletteHTTPException
from .db import SessionLocal, engine
from .errors import ApiError, error_response
from .models import Attachment, Chat, ChatMember, IdempotencyRecord, Message, MessageAttachment, User
from .schemas import (
AttachmentOut,
ChatOut,
ChatPage,
CreateChat,
CreateMessage,
CreateUser,
Health,
MessageOut,
MessagePage,
SessionOut,
UserOut,
)
from .settings import load_settings
from .storage import s3
settings = load_settings()
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
logger = logging.getLogger("messenger.api")
password_hasher = PasswordHasher(
time_cost=2,
memory_cost=19_456,
parallelism=1,
hash_len=32,
salt_len=16,
type=Type.ID,
)
DUMMY_PASSWORD_HASH = password_hasher.hash("dummy-password-for-timing-only")
redis_client = Redis.from_url(
settings.cache_url,
decode_responses=True,
socket_connect_timeout=2,
socket_timeout=2,
health_check_interval=30,
)
app = FastAPI(title="MTUSI Messenger — Lab 4", version="1.4.0")
BEARER_CHALLENGE = 'Bearer realm="messenger"'
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
TOKEN_RE = re.compile(r"^[A-Za-z0-9_-]{43}$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII)
# Docker assigns each container a stable hostname equal to its container ID.
# The load balancer forwards this value so tests can prove requests changed replicas.
INSTANCE_ID = socket.gethostname()
UPLOAD_LIMITER = asyncio.Semaphore(1)
def _iso(value: datetime) -> str:
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def _user_out(user: User) -> dict:
return {
"id": str(user.id),
"username": user.username,
"display_name": user.display_name,
"created_at": _iso(user.created_at),
}
def _message_out(message: Message, attachment_ids: list[uuid.UUID | str] | None = None) -> dict:
return {
"id": str(message.id),
"chat_id": str(message.chat_id),
"sender_id": str(message.sender_id),
"text": message.text,
"created_at": _iso(message.created_at),
"attachment_ids": [str(item) for item in (attachment_ids or [])],
}
def _attachment_out(attachment: Attachment) -> dict:
return {
"id": str(attachment.id),
"chat_id": str(attachment.chat_id),
"owner_id": str(attachment.owner_id),
"filename": attachment.filename,
"content_type": attachment.content_type,
"size_bytes": attachment.size_bytes,
"sha256": attachment.sha256,
"status": attachment.status,
"variants": attachment.variants or [],
"error_code": attachment.error_code,
"created_at": _iso(attachment.created_at),
"updated_at": _iso(attachment.updated_at),
}
def _message_attachment_ids(session, message_id: uuid.UUID) -> list[uuid.UUID]:
return session.scalars(
select(MessageAttachment.attachment_id)
.where(MessageAttachment.message_id == message_id)
.order_by(MessageAttachment.position)
).all()
def _validate_image(file_obj, claimed_content_type: str) -> str:
file_obj.seek(0)
try:
with Image.open(file_obj) as image:
detected = {
"PNG": "image/png",
"JPEG": "image/jpeg",
}.get(image.format or "")
if detected is None or detected != claimed_content_type:
raise _api_error("unsupported_media_type", "Формат изображения не поддерживается", 415)
width, height = image.size
if width < 1 or height < 1 or width * height > settings.max_image_pixels:
raise _api_error("payload_too_large", "Размер изображения превышает допустимый предел", 413)
image.verify()
file_obj.seek(0)
with Image.open(file_obj) as image:
image.load()
except ApiError:
raise
except (UnidentifiedImageError, OSError, SyntaxError, ValueError, Image.DecompressionBombError):
raise _api_error("unsupported_media_type", "Изображение повреждено или не поддерживается", 415) from None
finally:
file_obj.seek(0)
return claimed_content_type
def _valid_filename(filename: str) -> bool:
if not 1 <= len(filename) <= 255:
return False
if "/" in filename or "\\" in filename:
return False
return not any(unicodedata.category(char) == "Cc" for char in filename)
def _is_not_found_s3(exc: ClientError) -> bool:
code = str(exc.response.get("Error", {}).get("Code", ""))
return code in {"404", "NoSuchKey", "NoSuchBucket", "NotFound"}
def _chat_out(session, chat: Chat) -> dict:
members = session.scalars(
select(ChatMember.user_id)
.where(ChatMember.chat_id == chat.id)
.order_by(ChatMember.user_id)
).all()
return {
"id": str(chat.id),
"title": chat.title,
"member_ids": [str(member_id) for member_id in members],
"created_at": _iso(chat.created_at),
}
def _api_error(code: str, message: str, status: int, challenge: str | None = None) -> ApiError:
return ApiError(code, message, status, challenge)
def _unauthorized(challenge: str = BEARER_CHALLENGE) -> ApiError:
return _api_error("unauthorized", "Нет действительной идентификации", 401, challenge)
def _encode_cursor(payload: dict) -> str:
raw = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def _decode_cursor(value: str) -> dict:
try:
padded = value + ("=" * ((4 - len(value) % 4) % 4))
raw = base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True)
payload = json.loads(raw.decode("utf-8"))
if not isinstance(payload, dict):
raise ValueError("cursor must be an object")
return payload
except (ValueError, UnicodeError, binascii.Error, json.JSONDecodeError) as exc:
raise _api_error("invalid_request", "Неверный cursor", 400) from exc
def _cursor_position(payload: dict) -> tuple[datetime, uuid.UUID]:
try:
created_at = datetime.fromisoformat(payload["t"].replace("Z", "+00:00"))
if created_at.tzinfo is None:
raise ValueError("cursor time must include timezone")
item_id = uuid.UUID(payload["i"])
return created_at.astimezone(timezone.utc), item_id
except (KeyError, TypeError, ValueError, AttributeError) as exc:
raise _api_error("invalid_request", "Неверный cursor", 400) from exc
def _session_key(token: str) -> str:
digest = hashlib.sha256(token.encode("ascii")).hexdigest()
return "session:v1:" + digest
def _parse_bearer(request: Request) -> str:
header = request.headers.get("authorization", "")
scheme, separator, token = header.partition(" ")
if scheme.lower() != "bearer" or not separator or not TOKEN_RE.fullmatch(token):
raise _unauthorized()
return token
def _load_session(token: str) -> dict:
value = redis_client.get(_session_key(token))
if value is None:
raise _unauthorized()
try:
session = json.loads(value)
user_id = uuid.UUID(session["user_id"])
expires_at = datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00"))
if expires_at.tzinfo is None or datetime.now(timezone.utc) >= expires_at:
redis_client.delete(_session_key(token))
raise _unauthorized()
return {"user_id": user_id, "expires_at": expires_at}
except ApiError:
raise
except (KeyError, TypeError, ValueError, AttributeError, json.JSONDecodeError):
redis_client.delete(_session_key(token))
raise _unauthorized() from None
def _current_user(request: Request) -> tuple[User, str]:
token = _parse_bearer(request)
session_data = _load_session(token)
with SessionLocal() as session:
user = session.get(User, session_data["user_id"])
if user is None:
redis_client.delete(_session_key(token))
raise _unauthorized()
return user, token
def _parse_basic(header: str) -> tuple[str, str]:
scheme, separator, encoded = header.partition(" ")
if scheme.lower() != "basic" or not separator or not encoded:
raise _unauthorized(BASIC_CHALLENGE)
try:
decoded = base64.b64decode(encoded.encode("ascii"), validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except (UnicodeError, ValueError, binascii.Error):
raise _unauthorized(BASIC_CHALLENGE) from None
if not USERNAME_RE.fullmatch(username) or not password:
raise _unauthorized(BASIC_CHALLENGE)
return username, password
def _password_matches(password_hash: str, password: str) -> bool:
try:
return password_hasher.verify(password_hash, password)
except (VerifyMismatchError, VerificationError, InvalidHashError):
return False
def _create_session(username: str, password: str) -> dict:
with SessionLocal() as session:
user = session.scalar(select(User).where(User.username == username))
stored_hash = user.password_hash if user is not None else DUMMY_PASSWORD_HASH
password_ok = _password_matches(stored_hash, password)
if user is None or not password_ok:
raise _unauthorized(BASIC_CHALLENGE)
user_payload = _user_out(user)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.session_ttl_seconds)
for _ in range(3):
token = secrets.token_urlsafe(32)
stored_value = json.dumps(
{"user_id": user_payload["id"], "expires_at": _iso(expires_at)},
separators=(",", ":"),
)
if redis_client.set(_session_key(token), stored_value, ex=settings.session_ttl_seconds, nx=True):
return {
"token": token,
"token_type": "Bearer",
"expires_at": _iso(expires_at),
"user": user_payload,
}
logger.error("session_token_collision_limit_reached")
raise _api_error("unavailable", "Сервис временно недоступен", 503)
def _get_chat_for_member(session, chat_id: uuid.UUID, user_id: uuid.UUID) -> Chat | None:
return session.scalar(
select(Chat)
.join(ChatMember, ChatMember.chat_id == Chat.id)
.where(Chat.id == chat_id, ChatMember.user_id == user_id)
)
@app.middleware("http")
async def request_metadata_middleware(request: Request, call_next):
request_id = str(uuid.uuid4())
request.state.request_id = request_id
try:
response = await call_next(request)
except Exception as exc:
# Keep an unexpected failure bounded and ensure the API response still
# identifies the request and the replica that created it.
logger.error("internal_error request_id=%s type=%s", request_id, type(exc).__name__)
response = error_response(request, "internal_error", "Внутренняя ошибка", 500)
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
@app.exception_handler(ApiError)
async def handle_api_error(request: Request, exc: ApiError):
return error_response(request, exc.code, exc.message, exc.status, exc.challenge)
@app.exception_handler(RequestValidationError)
async def handle_validation_error(request: Request, _exc: RequestValidationError):
return error_response(request, "invalid_request", "Неверный запрос", 400)
@app.exception_handler(StarletteHTTPException)
async def handle_http_error(request: Request, exc: StarletteHTTPException):
if exc.status_code == 404:
return error_response(request, "not_found", "Объект не найден", 404)
return error_response(request, "invalid_request", "Неверный запрос", 400)
@app.exception_handler(OperationalError)
@app.exception_handler(SQLAlchemyTimeoutError)
async def handle_database_unavailable(request: Request, exc: Exception):
logger.warning("database_unavailable request_id=%s type=%s", request.state.request_id, type(exc).__name__)
return error_response(request, "unavailable", "Сервис временно недоступен", 503)
@app.exception_handler(RedisError)
async def handle_cache_unavailable(request: Request, exc: RedisError):
logger.warning("session_store_unavailable request_id=%s type=%s", request.state.request_id, type(exc).__name__)
return error_response(request, "unavailable", "Сервис временно недоступен", 503)
@app.exception_handler(ClientError)
@app.exception_handler(BotoCoreError)
async def handle_object_store_unavailable(request: Request, exc: Exception):
logger.warning("object_store_unavailable request_id=%s type=%s", request.state.request_id, type(exc).__name__)
return error_response(request, "unavailable", "Сервис временно недоступен", 503)
@app.exception_handler(DBAPIError)
async def handle_database_error(request: Request, exc: DBAPIError):
if exc.connection_invalidated:
logger.warning("database_connection_lost request_id=%s", request.state.request_id)
return error_response(request, "unavailable", "Сервис временно недоступен", 503)
logger.error("database_error request_id=%s type=%s", request.state.request_id, type(exc).__name__)
return error_response(request, "internal_error", "Внутренняя ошибка", 500)
@app.exception_handler(Exception)
async def handle_unexpected_error(request: Request, exc: Exception):
logger.error("internal_error request_id=%s type=%s", request.state.request_id, type(exc).__name__)
return error_response(request, "internal_error", "Внутренняя ошибка", 500)
@app.get("/health/live", response_model=Health)
def live():
return {"status": "ok"}
@app.get("/health/ready", response_model=Health, responses={503: {"description": "Dependency unavailable"}})
def ready():
with SessionLocal() as session:
session.execute(text("SELECT 1"))
redis_client.ping()
s3.get_bucket_location(Bucket=settings.s3_bucket)
return {"status": "ok"}
@app.post("/api/v1/users", response_model=UserOut, status_code=201)
def register_user(payload: CreateUser):
password_hash = password_hasher.hash(payload.password)
user = User(username=payload.username, display_name=payload.display_name, password_hash=password_hash)
try:
with SessionLocal.begin() as session:
session.add(user)
session.flush()
except DBAPIError as exc:
if getattr(getattr(exc, "orig", None), "sqlstate", None) == "23505":
raise _api_error("conflict", "username занят", 409) from exc
raise
return _user_out(user)
@app.post("/api/v1/auth/sessions", response_model=SessionOut, status_code=201)
async def create_session(request: Request):
if await request.body():
raise _api_error("invalid_request", "Basic authentication must not include a body", 400)
username, password = _parse_basic(request.headers.get("authorization", ""))
return await run_in_threadpool(_create_session, username, password)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def revoke_session(request: Request):
_user, token = _current_user(request)
redis_client.delete(_session_key(token))
return None
@app.get("/api/v1/users/me", response_model=UserOut)
def current_user(request: Request):
user, _token = _current_user(request)
return _user_out(user)
@app.post("/api/v1/chats", response_model=ChatOut, status_code=201)
def create_chat(payload: CreateChat, request: Request):
user, _token = _current_user(request)
member_ids = [uuid.UUID(value) for value in payload.member_ids]
if user.id not in member_ids:
raise _api_error("invalid_request", "Вызывающий должен входить в member_ids", 400)
try:
with SessionLocal.begin() as session:
chat = Chat(title=payload.title)
session.add(chat)
session.flush()
session.add_all(ChatMember(chat_id=chat.id, user_id=member_id) for member_id in member_ids)
session.flush()
result = _chat_out(session, chat)
except DBAPIError as exc:
if getattr(getattr(exc, "orig", None), "sqlstate", None) in {"23503", "23505"}:
raise _api_error("invalid_request", "Неизвестный участник", 400) from exc
raise
return result
@app.get("/api/v1/chats", response_model=ChatPage)
def list_chats(request: Request, limit: int = 50, cursor: str | None = None):
user, _token = _current_user(request)
if limit < 1 or limit > 100:
raise _api_error("invalid_request", "limit вне диапазона", 400)
statement = (
select(Chat)
.join(ChatMember, ChatMember.chat_id == Chat.id)
.where(ChatMember.user_id == user.id)
.order_by(Chat.created_at, Chat.id)
)
if cursor is not None:
payload = _decode_cursor(cursor)
if payload.get("u") != str(user.id):
raise _api_error("invalid_request", "Неверный cursor", 400)
created_at, chat_id = _cursor_position(payload)
statement = statement.where(tuple_(Chat.created_at, Chat.id) > tuple_(created_at, chat_id))
with SessionLocal() as session:
chats = session.scalars(statement.limit(limit + 1)).all()
has_more = len(chats) > limit
page = chats[:limit]
items = [_chat_out(session, chat) for chat in page]
next_cursor = None
if has_more:
last = page[-1]
next_cursor = _encode_cursor({"u": str(user.id), "t": _iso(last.created_at), "i": str(last.id)})
return {"items": items, "next_cursor": next_cursor}
@app.get("/api/v1/chats/{chat_id}", response_model=ChatOut)
def get_chat(chat_id: uuid.UUID, request: Request):
user, _token = _current_user(request)
with SessionLocal() as session:
chat = _get_chat_for_member(session, chat_id, user.id)
if chat is None:
raise _api_error("not_found", "Объект не найден", 404)
return _chat_out(session, chat)
@app.post("/api/v1/chats/{chat_id}/messages", response_model=MessageOut, status_code=201)
def create_message(
chat_id: uuid.UUID,
payload: CreateMessage,
request: Request,
idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"),
):
user, _token = _current_user(request)
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
raise _api_error("invalid_request", "Нужен корректный Idempotency-Key", 400)
if not payload.text.strip() and not payload.attachment_ids:
raise _api_error("invalid_request", "Сообщение не может состоять только из пробелов", 400)
# Hash the validated semantic JSON, not the raw bytes, so whitespace and
# object key order do not change the identity of a request.
canonical_body = json.dumps(
{"text": payload.text, "attachment_ids": payload.attachment_ids},
ensure_ascii=False, sort_keys=True, separators=(",", ":"),
)
body_hash = hashlib.sha256(canonical_body.encode("utf-8")).hexdigest()
legacy_hash = hashlib.sha256(
json.dumps({"text": payload.text}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
path = request.url.path
now = datetime.now(timezone.utc)
expires_at = now + timedelta(hours=24)
scope = (
IdempotencyRecord.user_id == user.id,
IdempotencyRecord.method == "POST",
IdempotencyRecord.path == path,
IdempotencyRecord.idempotency_key == idempotency_key,
)
with SessionLocal.begin() as session:
chat = _get_chat_for_member(session, chat_id, user.id)
if chat is None:
raise _api_error("not_found", "Объект не найден", 404)
attachment_ids = [uuid.UUID(value) for value in payload.attachment_ids]
if attachment_ids:
attached = session.scalars(
select(Attachment).where(
Attachment.chat_id == chat_id,
Attachment.id.in_(attachment_ids),
)
).all()
if len(attached) != len(attachment_ids):
raise _api_error("not_found", "Объект не найден", 404)
if any(item.status != "ready" for item in attached):
raise _api_error("conflict", "Вложение ещё не готово", 409)
# A key becomes reusable only after its 24-hour retention window.
session.execute(delete(IdempotencyRecord).where(*scope, IdempotencyRecord.expires_at <= now))
insert_record = (
pg_insert(IdempotencyRecord)
.values(
user_id=user.id,
method="POST",
path=path,
idempotency_key=idempotency_key,
request_hash=body_hash,
response_body={},
expires_at=expires_at,
)
.on_conflict_do_nothing(constraint="idempotency_records_scope_key_uq")
.returning(IdempotencyRecord.id)
)
record_id = session.execute(insert_record).scalar_one_or_none()
if record_id is None:
# A concurrent insert waits on the unique constraint. At READ
# COMMITTED the winning transaction is visible to this SELECT.
existing = session.scalar(select(IdempotencyRecord).where(*scope))
if existing is None:
raise _api_error("unavailable", "Сервис временно недоступен", 503)
if existing.request_hash != body_hash and not (not attachment_ids and existing.request_hash == legacy_hash):
raise _api_error("conflict", "Idempotency-Key уже использован для другого запроса", 409)
result = dict(existing.response_body)
result.setdefault("attachment_ids", [])
else:
message = Message(chat_id=chat_id, sender_id=user.id, text=payload.text)
session.add(message)
session.flush()
session.add_all(
MessageAttachment(
message_id=message.id,
attachment_id=attachment_id,
chat_id=chat_id,
position=position,
)
for position, attachment_id in enumerate(attachment_ids)
)
result = _message_out(message, attachment_ids)
session.execute(
update(IdempotencyRecord)
.where(IdempotencyRecord.id == record_id)
.values(response_body=result)
)
return result
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
def list_messages(chat_id: uuid.UUID, request: Request, limit: int = 50, cursor: str | None = None):
user, _token = _current_user(request)
with SessionLocal() as session:
chat = _get_chat_for_member(session, chat_id, user.id)
if chat is None:
raise _api_error("not_found", "Объект не найден", 404)
if limit < 1 or limit > 100:
raise _api_error("invalid_request", "limit вне диапазона", 400)
after = None
if cursor is not None:
payload = _decode_cursor(cursor)
if payload.get("c") != str(chat_id):
raise _api_error("invalid_request", "cursor от другого чата", 400)
after = _cursor_position(payload)
statement = select(Message).where(Message.chat_id == chat_id)
if after is not None:
statement = statement.where(tuple_(Message.created_at, Message.id) > tuple_(*after))
statement = statement.order_by(Message.created_at, Message.id).limit(limit + 1)
rows = session.scalars(statement).all()
has_more = len(rows) > limit
page = rows[:limit]
attachment_rows = session.execute(
select(MessageAttachment.message_id, MessageAttachment.attachment_id)
.where(MessageAttachment.message_id.in_([message.id for message in page]))
.order_by(MessageAttachment.message_id, MessageAttachment.position)
).all() if page else []
ids_by_message: dict[uuid.UUID, list[uuid.UUID]] = {}
for message_id, attachment_id in attachment_rows:
ids_by_message.setdefault(message_id, []).append(attachment_id)
items = [_message_out(message, ids_by_message.get(message.id, [])) for message in page]
next_cursor = None
if has_more:
last = page[-1]
next_cursor = _encode_cursor({"c": str(chat_id), "t": _iso(last.created_at), "i": str(last.id)})
return {"items": items, "next_cursor": next_cursor}
@app.post(
"/api/v1/chats/{chat_id}/attachments",
response_model=AttachmentOut,
status_code=201,
)
async def upload_attachment(
chat_id: uuid.UUID,
request: Request,
filename: str,
):
user, _token = _current_user(request)
with SessionLocal() as session:
if _get_chat_for_member(session, chat_id, user.id) is None:
raise _api_error("not_found", "Объект не найден", 404)
if not _valid_filename(filename):
raise _api_error("invalid_request", "Неверное имя файла", 400)
content_type_header = request.headers.get("content-type", "")
header = EmailMessage()
try:
header["Content-Type"] = content_type_header
claimed_type = header.get_content_type().lower()
except (TypeError, ValueError):
claimed_type = ""
if claimed_type not in {"image/png", "image/jpeg"}:
raise _api_error("unsupported_media_type", "Поддерживаются только PNG и JPEG", 415)
content_length = request.headers.get("content-length")
if content_length is not None:
try:
if int(content_length) > settings.max_upload_bytes:
raise _api_error("payload_too_large", "Файл превышает допустимый размер", 413)
except ValueError:
raise _api_error("invalid_request", "Неверный Content-Length", 400) from None
async with UPLOAD_LIMITER:
with tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b", dir="/tmp") as temporary:
size_bytes = 0
digest = hashlib.sha256()
async for chunk in request.stream():
if not chunk:
continue
size_bytes += len(chunk)
if size_bytes > settings.max_upload_bytes:
raise _api_error("payload_too_large", "Файл превышает допустимый размер", 413)
digest.update(chunk)
temporary.write(chunk)
if size_bytes == 0:
raise _api_error("unsupported_media_type", "Файл пуст", 415)
detected_type = await run_in_threadpool(_validate_image, temporary, claimed_type)
attachment_id = uuid.uuid4()
object_key = f"originals/{attachment_id.hex}"
temporary.seek(0)
try:
await run_in_threadpool(
s3.put_object,
Bucket=settings.s3_bucket,
Key=object_key,
Body=temporary,
ContentLength=size_bytes,
ContentType=detected_type,
Metadata={"sha256": digest.hexdigest()},
)
except (ClientError, BotoCoreError):
try:
await run_in_threadpool(s3.delete_object, Bucket=settings.s3_bucket, Key=object_key)
except Exception as cleanup_exc:
logger.warning(
"attachment_compensation_failed request_id=%s type=%s",
request.state.request_id,
type(cleanup_exc).__name__,
)
raise _api_error("unavailable", "Сервис временно недоступен", 503) from None
try:
with SessionLocal.begin() as session:
attachment = Attachment(
id=attachment_id,
chat_id=chat_id,
owner_id=user.id,
filename=filename,
content_type=detected_type,
size_bytes=size_bytes,
sha256=digest.hexdigest(),
status="ready",
variants=[],
error_code=None,
object_key=object_key,
)
session.add(attachment)
session.flush()
result = _attachment_out(attachment)
except Exception:
try:
await run_in_threadpool(s3.delete_object, Bucket=settings.s3_bucket, Key=object_key)
except Exception as cleanup_exc:
logger.warning(
"attachment_compensation_failed request_id=%s type=%s",
request.state.request_id,
type(cleanup_exc).__name__,
)
raise
return result
@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentOut)
def get_attachment(attachment_id: uuid.UUID, request: Request):
user, _token = _current_user(request)
with SessionLocal() as session:
attachment = session.scalar(
select(Attachment)
.join(ChatMember, ChatMember.chat_id == Attachment.chat_id)
.where(Attachment.id == attachment_id, ChatMember.user_id == user.id)
)
if attachment is None:
raise _api_error("not_found", "Объект не найден", 404)
return _attachment_out(attachment)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(
attachment_id: uuid.UUID,
request: Request,
variant: str = "original",
):
user, _token = _current_user(request)
with SessionLocal() as session:
attachment = session.scalar(
select(Attachment)
.join(ChatMember, ChatMember.chat_id == Attachment.chat_id)
.where(Attachment.id == attachment_id, ChatMember.user_id == user.id)
)
if attachment is None:
raise _api_error("not_found", "Объект не найден", 404)
if attachment.status != "ready":
raise _api_error("conflict", "Вложение ещё не готово", 409)
if variant != "original":
raise _api_error("not_found", "Объект не найден", 404)
object_key = attachment.object_key
content_type = attachment.content_type
size_bytes = attachment.size_bytes
try:
stored = await run_in_threadpool(s3.get_object, Bucket=settings.s3_bucket, Key=object_key)
except ClientError as exc:
if _is_not_found_s3(exc):
logger.error("attachment_object_missing request_id=%s", request.state.request_id)
raise _api_error("unavailable", "Сервис временно недоступен", 503) from None
raise _api_error("unavailable", "Сервис временно недоступен", 503) from None
except BotoCoreError:
raise _api_error("unavailable", "Сервис временно недоступен", 503) from None
body = stored["Body"]
async def chunks():
try:
while True:
chunk = await run_in_threadpool(body.read, 64 * 1024)
if not chunk:
break
yield chunk
finally:
await run_in_threadpool(body.close)
return StreamingResponse(
chunks(),
media_type=content_type,
headers={
"Content-Length": str(size_bytes),
"Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff",
},
)
+36
View File
@@ -0,0 +1,36 @@
from pathlib import Path
from sqlalchemy import text
from .db import engine
MIGRATIONS = Path(__file__).resolve().parents[2] / "migrations"
def migrate() -> None:
with engine.begin() as connection:
connection.exec_driver_sql("SELECT pg_advisory_xact_lock(2026102202)")
connection.exec_driver_sql(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(version VARCHAR(128) PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP)"
)
applied = set(connection.execute(text("SELECT version FROM schema_migrations")).scalars())
for path in sorted(MIGRATIONS.glob("*.sql")):
version = path.stem
if version in applied:
continue
sql = path.read_text(encoding="utf-8")
for statement in sql.split(";"):
if statement.strip():
connection.exec_driver_sql(statement)
connection.execute(
text("INSERT INTO schema_migrations(version) VALUES (:version)"),
{"version": version},
)
print(f"Applied migration {version}")
engine.dispose()
if __name__ == "__main__":
migrate()
+145
View File
@@ -0,0 +1,145 @@
import uuid
from datetime import datetime
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, ForeignKeyConstraint, Index, Integer, String, Text, UniqueConstraint, func
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
pass
class User(Base):
__tablename__ = "users"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
username: Mapped[str] = mapped_column(String(32), nullable=False, unique=True)
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
class Chat(Base):
__tablename__ = "chats"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
title: Mapped[str] = mapped_column(String(100), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
class ChatMember(Base):
__tablename__ = "chat_members"
__table_args__ = (
Index("chat_members_user_id_idx", "user_id", "chat_id"),
)
chat_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
)
user_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("users.id", ondelete="RESTRICT"), primary_key=True
)
class Message(Base):
__tablename__ = "messages"
__table_args__ = (
ForeignKeyConstraint(
["chat_id", "sender_id"],
["chat_members.chat_id", "chat_members.user_id"],
name="messages_sender_is_chat_member_fk",
ondelete="CASCADE",
),
UniqueConstraint("id", "chat_id", name="messages_id_chat_id_uq"),
Index("messages_history_idx", "chat_id", "created_at", "id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
sender_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
text: Mapped[str] = mapped_column(String(4000), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
class IdempotencyRecord(Base):
"""Shared, transactional result cache for retried message POST requests."""
__tablename__ = "idempotency_records"
__table_args__ = (
UniqueConstraint(
"user_id", "method", "path", "idempotency_key",
name="idempotency_records_scope_key_uq",
),
Index("idempotency_records_expires_at_idx", "expires_at"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
user_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
method: Mapped[str] = mapped_column(String(8), nullable=False)
path: Mapped[str] = mapped_column(String(200), nullable=False)
idempotency_key: Mapped[str] = mapped_column(String(128), nullable=False)
request_hash: Mapped[str] = mapped_column(String(64), nullable=False)
response_body: Mapped[dict] = mapped_column(JSONB, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class Attachment(Base):
__tablename__ = "attachments"
__table_args__ = (
ForeignKeyConstraint(
["chat_id", "owner_id"],
["chat_members.chat_id", "chat_members.user_id"],
name="attachments_owner_is_chat_member_fk",
ondelete="CASCADE",
),
UniqueConstraint("id", "chat_id", name="attachments_id_chat_id_uq"),
CheckConstraint("char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_length_ck"),
CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_ck"),
CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_ck"),
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="attachments_sha256_ck"),
CheckConstraint("status = 'ready'", name="attachments_status_ck"),
Index("attachments_chat_created_idx", "chat_id", "created_at", "id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
owner_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
filename: Mapped[str] = mapped_column(String(255), nullable=False)
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False)
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
status: Mapped[str] = mapped_column(String(16), nullable=False, default="ready")
variants: Mapped[list] = mapped_column(JSONB, nullable=False, default=list, server_default="[]")
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
object_key: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now())
class MessageAttachment(Base):
__tablename__ = "message_attachments"
__table_args__ = (
ForeignKeyConstraint(
["message_id", "chat_id"],
["messages.id", "messages.chat_id"],
name="message_attachments_message_chat_fk",
ondelete="CASCADE",
),
ForeignKeyConstraint(
["attachment_id", "chat_id"],
["attachments.id", "attachments.chat_id"],
name="message_attachments_attachment_chat_fk",
ondelete="CASCADE",
),
UniqueConstraint("message_id", "position", name="message_attachments_position_uq"),
Index("message_attachments_attachment_idx", "attachment_id"),
)
message_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
attachment_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False)
position: Mapped[int] = mapped_column(Integer, nullable=False)
+60
View File
@@ -0,0 +1,60 @@
"""Find orphan objects older than a safety grace period; deletion requires --apply."""
import argparse
from datetime import datetime, timedelta, timezone
import os
import boto3
from botocore.config import Config
from sqlalchemy import select
from .db import SessionLocal
from .models import Attachment
from .settings import _secret
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--apply", action="store_true", help="delete confirmed stale orphans")
args = parser.parse_args()
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/")
region = os.environ.get("S3_REGION", "us-east-1")
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
try:
grace_seconds = int(os.environ.get("ORPHAN_GRACE_SECONDS", "86400"))
except ValueError as exc:
raise SystemExit("ORPHAN_GRACE_SECONDS must be an integer") from exc
if grace_seconds < 3600:
raise SystemExit("ORPHAN_GRACE_SECONDS must be at least 3600 seconds")
client = boto3.client(
"s3",
endpoint_url=endpoint,
region_name=region,
aws_access_key_id=_secret("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY"),
aws_secret_access_key=_secret("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY"),
config=Config(connect_timeout=2, read_timeout=5, retries={"total_max_attempts": 1}, s3={"addressing_style": "path"}),
)
with SessionLocal() as session:
known_keys = set(session.scalars(select(Attachment.object_key)).all())
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
scanned = stale = deleted = 0
paginator = client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=bucket, Prefix="originals/"):
for obj in page.get("Contents", []):
scanned += 1
key = obj["Key"]
if key in known_keys or obj.get("LastModified") is None or obj["LastModified"] > cutoff:
continue
stale += 1
if args.apply:
client.delete_object(Bucket=bucket, Key=key)
deleted += 1
print(
f"orphan_reconciliation mode={'apply' if args.apply else 'dry-run'} "
f"grace_seconds={grace_seconds} scanned={scanned} stale={stale} deleted={deleted}"
)
if __name__ == "__main__":
main()
+196
View File
@@ -0,0 +1,196 @@
"""Idempotently create the private bucket and a bucket-scoped API service account."""
import json
import os
import time
from urllib.error import HTTPError, URLError
from urllib.parse import quote
from urllib.request import Request, urlopen
import boto3
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.config import Config
from botocore.credentials import Credentials
from botocore.exceptions import ClientError
from .settings import _secret
def _admin_request(method: str, path: str, payload: dict | None = None) -> tuple[int, str | None]:
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/")
region = os.environ.get("S3_REGION", "us-east-1")
access = _secret("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY")
secret = _secret("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY")
body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") if payload is not None else None
url = endpoint + path
# RustFS 1.0.5's admin API may reject a signed GET with an empty body
# when Content-Length is omitted by urllib. Sign and send the explicit
# zero length so the request is unambiguous to its SigV4 verifier.
headers = {"Content-Type": "application/json"} if body is not None else {"Content-Length": "0"}
signed = AWSRequest(method=method, url=url, data=body, headers=headers)
S3SigV4Auth(Credentials(access, secret), "s3", region).add_auth(signed)
request = Request(url, data=body, headers=dict(signed.headers), method=method)
try:
with urlopen(request, timeout=3) as response:
response.read(1024 * 1024)
return response.status, None
except HTTPError as exc:
raw = exc.read(1024 * 1024)
detail = ""
try:
document = json.loads(raw)
except (json.JSONDecodeError, UnicodeDecodeError):
document = None
if isinstance(document, dict):
detail = "; ".join(
str(document[key])
for key in ("Code", "code", "Message", "message", "error")
if document.get(key)
)
if not detail:
detail = raw.decode("utf-8", errors="replace")
detail = " ".join(detail.split())[:240]
# RustFS can echo request details in an error response. Redact every
# credential before including a short diagnostic in the container log.
for file_variable, value_variable in (
("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY"),
("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY"),
("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"),
("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"),
):
try:
credential = _secret(file_variable, value_variable)
except RuntimeError:
continue
detail = detail.replace(credential, "[redacted]")
detail = detail.replace(quote(credential, safe=""), "[redacted]")
return exc.code, detail or None
def _failure(action: str, status: int, detail: str | None) -> RuntimeError:
suffix = f": {detail}" if detail else ""
return RuntimeError(f"RustFS API service-account {action} failed with HTTP {status}{suffix}")
def _create_api_service_account(endpoint: str, region: str, bucket: str) -> None:
access_key = _secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID")
secret_key = _secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY")
policy = {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "BucketLocation",
"Effect": "Allow",
"Action": ["s3:GetBucketLocation"],
"Resource": [f"arn:aws:s3:::{bucket}"],
},
{
"Sid": "AttachmentObjectsOnly",
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": [f"arn:aws:s3:::{bucket}/*"],
},
],
}
encoded_key = quote(access_key, safe="")
info_path = f"/rustfs/admin/v3/info-service-account?accessKey={encoded_key}"
status, detail = _admin_request("GET", info_path)
if status == 404:
status, detail = _admin_request(
"PUT",
"/rustfs/admin/v3/add-service-account",
{
"accessKey": access_key,
"secretKey": secret_key,
"name": "messenger-api",
"description": "Limited access to the private attachment bucket",
"policy": policy,
},
)
if status not in (200, 201, 204):
raise _failure("creation", status, detail)
elif status != 200:
raise _failure("lookup", status, detail)
else:
update_path = f"/rustfs/admin/v3/update-service-account?accessKey={encoded_key}"
status, detail = _admin_request(
"POST",
update_path,
{"policy": policy, "name": "messenger-api", "description": "Limited access to the private attachment bucket"},
)
if status not in (200, 204):
raise _failure("policy update", status, detail)
client = boto3.client(
"s3",
endpoint_url=endpoint,
region_name=region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=Config(connect_timeout=2, read_timeout=3, retries={"total_max_attempts": 1}, s3={"addressing_style": "path"}),
)
client.get_bucket_location(Bucket=bucket)
try:
response = client.list_buckets()
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code not in {"AccessDenied", "403"}:
raise RuntimeError("RustFS API credentials were not explicitly denied ListAllMyBuckets") from None
else:
# RustFS follows MinIO-compatible filtered listing: ListBuckets may
# succeed while returning only buckets this identity can access.
# Treat unrelated visible buckets as a leak, not the filtered result.
visible = {item.get("Name") for item in response.get("Buckets", [])}
if visible - {bucket}:
raise RuntimeError("RustFS API credentials can see buckets outside the attachment bucket")
def main() -> None:
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/")
region = os.environ.get("S3_REGION", "us-east-1")
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
root_access = _secret("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY")
root_secret = _secret("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY")
root = boto3.client(
"s3",
endpoint_url=endpoint,
region_name=region,
aws_access_key_id=root_access,
aws_secret_access_key=root_secret,
config=Config(connect_timeout=2, read_timeout=3, retries={"total_max_attempts": 1}, s3={"addressing_style": "path"}),
)
last_error = None
for _ in range(60):
try:
try:
root.head_bucket(Bucket=bucket)
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code in {"404", "NoSuchBucket", "NotFound"}:
root.create_bucket(Bucket=bucket)
else:
raise
break
except Exception as exc:
last_error = exc
time.sleep(1)
else:
raise RuntimeError("RustFS did not become ready for bucket bootstrap") from last_error
anonymous_url = f"{endpoint}/{quote(bucket, safe='')}"
try:
with urlopen(Request(anonymous_url, method="GET"), timeout=2) as response:
if response.status < 400:
raise RuntimeError("The attachment bucket allows anonymous access")
except HTTPError as exc:
if exc.code != 403:
raise RuntimeError(f"Anonymous bucket check returned unexpected HTTP {exc.code}") from None
except URLError as exc:
raise RuntimeError("Could not verify private bucket access") from exc
_create_api_service_account(endpoint, region, bucket)
print("PASS: private attachment bucket and least-privilege API credentials are ready.")
if __name__ == "__main__":
main()
+123
View File
@@ -0,0 +1,123 @@
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator
import uuid
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class CreateUser(StrictModel):
username: StrictStr = Field(min_length=3, max_length=32, pattern=r"^[a-z0-9_]{3,32}$")
display_name: StrictStr = Field(min_length=1, max_length=100)
password: StrictStr = Field(min_length=12, max_length=128)
@field_validator("display_name")
@classmethod
def display_name_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("display_name must contain a non-whitespace character")
return value
class UserOut(BaseModel):
id: str
username: str
display_name: str
created_at: str
class CreateChat(StrictModel):
title: StrictStr = Field(min_length=1, max_length=100)
member_ids: list[StrictStr] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def title_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("title must contain a non-whitespace character")
return value
@field_validator("member_ids")
@classmethod
def members_are_unique_uuids(cls, values: list[str]) -> list[str]:
import uuid
try:
normalized = [str(uuid.UUID(value)) for value in values]
except (ValueError, AttributeError, TypeError) as exc:
raise ValueError("member_ids must contain UUIDs") from exc
if len(normalized) != len(set(normalized)):
raise ValueError("member_ids must be unique")
return normalized
class ChatOut(BaseModel):
id: str
title: str
member_ids: list[str]
created_at: str
class ChatPage(BaseModel):
items: list[ChatOut]
next_cursor: str | None
class CreateMessage(StrictModel):
text: StrictStr = Field(max_length=4000)
attachment_ids: list[StrictStr] = Field(default_factory=list, max_length=10)
@field_validator("attachment_ids")
@classmethod
def attachment_ids_are_unique_uuids(cls, values: list[str]) -> list[str]:
try:
normalized = [str(uuid.UUID(value)) for value in values]
except (ValueError, AttributeError, TypeError) as exc:
raise ValueError("attachment_ids must contain UUIDs") from exc
if len(normalized) != len(set(normalized)):
raise ValueError("attachment_ids must be unique")
return normalized
class MessageOut(BaseModel):
id: str
chat_id: str
sender_id: str
text: str
created_at: str
attachment_ids: list[str]
class AttachmentOut(BaseModel):
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
status: str = "ready"
variants: list[dict] = Field(default_factory=list)
error_code: str | None = None
created_at: str
updated_at: str
class MessagePage(BaseModel):
items: list[MessageOut]
next_cursor: str | None
class SessionOut(BaseModel):
token: str
token_type: str = "Bearer"
expires_at: str
user: UserOut
class Health(BaseModel):
status: str = "ok"
class ErrorBody(BaseModel):
error: dict[str, str]
+67
View File
@@ -0,0 +1,67 @@
from dataclasses import dataclass
import os
from pathlib import Path
def _secret(file_variable: str, value_variable: str) -> str:
file_path = os.environ.get(file_variable)
if file_path:
value = Path(file_path).read_text(encoding="utf-8").strip()
else:
value = os.environ.get(value_variable, "").strip()
if not value:
raise RuntimeError(f"Required setting {file_variable} is missing")
return value
def load_database_url() -> str:
return _secret("DATABASE_URL_FILE", "DATABASE_URL")
@dataclass(frozen=True)
class Settings:
cache_url: str
session_ttl_seconds: int
s3_endpoint: str
s3_region: str
s3_bucket: str
s3_access_key_id: str
s3_secret_access_key: str
max_upload_bytes: int
max_image_pixels: int
def load_settings() -> Settings:
try:
ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
except ValueError as exc:
raise RuntimeError("SESSION_TTL_SECONDS must be an integer") from exc
if ttl < 1 or ttl > 31_536_000:
raise RuntimeError("SESSION_TTL_SECONDS must be between 1 and 31536000")
try:
max_upload_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
max_image_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
except ValueError as exc:
raise RuntimeError("MAX_UPLOAD_BYTES and MAX_IMAGE_PIXELS must be integers") from exc
if not 1 <= max_upload_bytes <= 10_485_760:
raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10485760")
if not 1 <= max_image_pixels <= 40_000_000:
raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40000000")
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").strip()
region = os.environ.get("S3_REGION", "us-east-1").strip()
bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip()
if not endpoint.startswith("http://") and not endpoint.startswith("https://"):
raise RuntimeError("S3_ENDPOINT must be an HTTP(S) URL")
if not region or not bucket:
raise RuntimeError("S3_REGION and S3_BUCKET are required")
return Settings(
cache_url=_secret("CACHE_URL_FILE", "CACHE_URL"),
session_ttl_seconds=ttl,
s3_endpoint=endpoint.rstrip("/"),
s3_region=region,
s3_bucket=bucket,
s3_access_key_id=_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"),
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"),
max_upload_bytes=max_upload_bytes,
max_image_pixels=max_image_pixels,
)
+20
View File
@@ -0,0 +1,20 @@
from botocore.config import Config
import boto3
from .settings import load_settings
settings = load_settings()
s3 = boto3.client(
"s3",
endpoint_url=settings.s3_endpoint,
region_name=settings.s3_region,
aws_access_key_id=settings.s3_access_key_id,
aws_secret_access_key=settings.s3_secret_access_key,
config=Config(
connect_timeout=1,
read_timeout=2,
retries={"total_max_attempts": 1, "mode": "standard"},
s3={"addressing_style": "path"},
),
)
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Grade-4 attachment rejection checks; uses only the public HTTP contract."""
import uuid
import json
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, arguments, check, run
def main():
args = arguments("Lab 4 grade-4 negative attachment checks")
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
base_path = "/api/v1/chats/" + chat["id"] + "/attachments"
valid_png = (Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png").read_bytes()
too_large = b"\x00" * (10 * 1024 * 1024 + 1)
oversized = client.call(
"POST", base_path + "?filename=large.png", 413,
actor=alice, raw=too_large, headers={"Content-Type": "image/png"},
)
corrupt = client.call(
"POST", base_path + "?filename=broken.png", 415,
actor=alice, raw=b"not a PNG", headers={"Content-Type": "image/png"},
)
spoofed = client.call(
"POST", base_path + "?filename=spoofed.jpg", 415,
actor=alice, raw=valid_png, headers={"Content-Type": "image/jpeg"},
)
unsafe_name = client.call(
"POST", base_path + "?filename=..%2Foutside.png", 400,
actor=alice, raw=valid_png, headers={"Content-Type": "image/png"},
)
client.attachment(alice, chat)
second_chat = client.call(
"POST", "/api/v1/chats", 201, actor=bob,
body={"title": "Другой чат", "member_ids": [bob["id"], eve["id"]]}, schema="Chat",
)["json"]
foreign_attachment, _ = client.attachment(bob, second_chat)
foreign = client.call(
"POST", "/api/v1/chats/" + chat["id"] + "/messages", 404,
actor=alice,
body={"text": "чужое вложение", "attachment_ids": [foreign_attachment["id"]]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
unknown = client.call(
"POST", "/api/v1/chats/" + chat["id"] + "/messages", 404,
actor=alice,
body={"text": "неизвестное вложение", "attachment_ids": [str(uuid.uuid4())]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
summary = {
"oversized_upload": oversized["status"],
"corrupt_image": corrupt["status"],
"mime_spoof": spoofed["status"],
"unsafe_filename": unsafe_name["status"],
"foreign_chat_attachment": foreign["status"],
"unknown_attachment": unknown["status"],
"result": "PASS",
}
evidence = Path(__file__).resolve().parents[1] / "evidence/grade4-attachments.json"
evidence.parent.mkdir(parents=True, exist_ok=True)
evidence.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
print(json.dumps(summary, separators=(",", ":")))
if __name__ == "__main__":
run(main)
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Prove shared message idempotency across concurrent API replicas."""
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import sys
import threading
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, arguments, check, run
def main():
def configure(parser):
parser.add_argument("--workers", type=int, default=12)
parser.add_argument("--requests", type=int, default=24)
args = arguments("Concurrent duplicate message POST across API replicas", configure)
check(args.workers >= 2 and args.requests >= args.workers, "requests must be >= workers >= 2")
setup = Client(args.base_url, args.ca_file, timeout=10)
alice, bob, _eve, chat = setup.bootstrap()
key = uuid.uuid4().hex
body = "parallel-idempotency-" + uuid.uuid4().hex
barrier = threading.Barrier(args.workers)
def send(index):
client = Client(args.base_url, args.ca_file, timeout=10)
if index < args.workers:
barrier.wait(timeout=10)
result = client.message(alice, chat, body, key=key)
return result["json"], result["headers"]["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=args.workers) as pool:
results = list(pool.map(send, range(args.requests)))
messages = [item[0] for item in results]
instances = {item[1] for item in results}
check(all(message == messages[0] for message in messages), "same key/body returned different messages")
check(len(instances) >= 2, "concurrent requests did not reach two API replicas")
history = setup.call(
"GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100",
actor=bob, schema="MessagePage",
)["json"]["items"]
stored = [item for item in history if item["text"] == body]
check(len(stored) == 1, "concurrent duplicate created zero or multiple stored messages")
print(json.dumps({
"parallel_requests": args.requests,
"workers": args.workers,
"replicas_observed": len(instances),
"unique_message_ids": len({item["id"] for item in messages}),
"stored_copies": len(stored),
"result": "PASS",
}, ensure_ascii=False))
if __name__ == "__main__":
run(main)
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Stop RustFS briefly and confirm the API fails closed, then restore it."""
import subprocess
import sys
import time
import json
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run
def compose(*args):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=30,
)
def main():
args = arguments("Bounded S3 outage check; stops and restores only the local RustFS service")
client = Client(args.base_url, args.ca_file, timeout=12)
alice, _bob, _eve, chat = client.bootstrap()
stopped = False
results = {}
try:
compose("stop", "s3")
stopped = True
live = client.call("GET", "/health/live", schema="Health")
check(live["json"]["status"] == "ok", "liveness failed during S3 outage")
results["liveness"] = live["status"]
results["readiness_during_outage"] = client.call("GET", "/health/ready", 503)["status"]
image = (Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png").read_bytes()
upload = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=s3-outage.png",
503,
actor=alice,
raw=image,
headers={"Content-Type": "image/png"},
)
# urllib response headers are an HTTPMessage, not JSON serializable.
# Record only the safe scalar evidence needed for this outage check.
results["upload_during_outage"] = {
"status": upload["status"],
"error_code": upload["json"]["error"]["code"],
"elapsed_seconds": round(upload["elapsed"], 3),
}
finally:
if stopped:
compose("start", "s3")
deadline = time.monotonic() + 90
while True:
try:
client.call("GET", "/health/ready", schema="Health")
break
except Failure:
if time.monotonic() >= deadline:
raise
time.sleep(1)
print("PASS: RustFS is ready again.")
results["readiness_after_restore"] = 200
results["result"] = "PASS"
evidence = Path(__file__).resolve().parents[1] / "evidence/s3-outage.json"
evidence.parent.mkdir(parents=True, exist_ok=True)
evidence.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8")
print(json.dumps(results, separators=(",", ":")))
if __name__ == "__main__":
run(main)
+129
View File
@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""Upload ten decoded 3000x3000 PNGs concurrently and sample Docker API memory."""
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
import json
from pathlib import Path
import re
import struct
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import zlib
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, check
def png_chunk(kind: bytes, payload: bytes) -> bytes:
return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
def large_decode_png(width=3000, height=3000) -> bytes:
row = b"\x00" + (b"\x28\x80\xc0" * width)
compressor = zlib.compressobj(level=6)
compressed = bytearray()
for _ in range(height):
compressed.extend(compressor.compress(row))
compressed.extend(compressor.flush())
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", header) + png_chunk(b"IDAT", bytes(compressed)) + png_chunk(b"IEND", b"")
def memory_mib(containers):
result = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.ID}}|{{.MemUsage}}", *containers],
check=True, capture_output=True, text=True, timeout=10,
)
samples = {}
for line in result.stdout.splitlines():
fields = line.split("|", 1)
if len(fields) != 2:
continue
number = fields[1].split("/", 1)[0].strip().split()[0]
match = re.fullmatch(r"([0-9.]+)(B|KiB|MiB|GiB)", number)
if not match:
continue
value = float(match.group(1))
value_mib = value * {"GiB": 1024, "MiB": 1, "KiB": 1 / 1024, "B": 1 / (1024 * 1024)}[match.group(2)]
samples[fields[0]] = value_mib
return samples
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default=None)
parser.add_argument("--ca-file", default=None)
parser.add_argument("--concurrency", type=int, default=10)
parser.add_argument("--output", default="evidence/upload-memory.json")
args = parser.parse_args()
check(args.concurrency >= 10, "grade 4 memory run requires at least 10 concurrent clients")
client = Client(args.base_url, args.ca_file, timeout=90)
alice, _bob, _eve, chat = client.bootstrap()
image = large_decode_png()
containers = subprocess.check_output(
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], text=True, timeout=10,
).split()
check(len(containers) >= 2, "start at least two API replicas behind the load balancer")
before = memory_mib(containers)
peak = dict(before)
stop = threading.Event()
def sample():
while not stop.is_set():
try:
for container, value in memory_mib(containers).items():
peak[container] = max(peak.get(container, 0), value)
except (subprocess.SubprocessError, OSError):
pass
stop.wait(0.25)
monitor = threading.Thread(target=sample, daemon=True)
monitor.start()
path = f"/api/v1/chats/{chat['id']}/attachments?filename=memory-probe.png"
def upload(_index):
request = urllib.request.Request(
client.base + path,
data=image,
headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"},
method="POST",
)
try:
with client.opener.open(request, timeout=90) as response:
response.read(1024 * 1024)
check(response.status == 201, "upload did not return 201")
return response.headers.get("X-Instance-Id", "")
except urllib.error.HTTPError as exc:
raise AssertionError(f"upload returned HTTP {exc.code}") from None
started = time.monotonic()
successes = []
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = [pool.submit(upload, i) for i in range(args.concurrency)]
for future in as_completed(futures):
successes.append(future.result())
finally:
stop.set()
monitor.join(timeout=3)
report = {
"concurrent_clients": args.concurrency,
"successful_uploads": len(successes),
"decoded_dimensions": [3000, 3000],
"input_bytes": len(image),
"container_memory_mib_before": before,
"container_memory_mib_peak": peak,
"elapsed_seconds": round(time.monotonic() - started, 3),
"observed_instances": sorted({item for item in successes if item}),
}
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, separators=(",", ":")))
if __name__ == "__main__":
main()