ЛР 4 — БПИ2301 — Возжин — на 5 #70
+13
-9
@@ -4,17 +4,21 @@ APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
# Synthetic local credentials: replace them in your untracked .env.
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change-me-local
|
||||
POSTGRES_DB=messenger
|
||||
DATABASE_URL=postgresql+asyncpg://messenger:change-me-local@postgres:5432/messenger
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
|
||||
MINIO_ROOT_USER=local-minio-admin
|
||||
MINIO_ROOT_PASSWORD=change-me-minio-root
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=messenger-api
|
||||
S3_SECRET_ACCESS_KEY=change-me-minio-api
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
LOG_LEVEL=info
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
FROM python:3.13.15-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN python -m pip install -r requirements.txt \
|
||||
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app
|
||||
COPY app ./app
|
||||
COPY alembic ./alembic
|
||||
COPY alembic.ini .
|
||||
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=3).read()"]
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"]
|
||||
@@ -0,0 +1,15 @@
|
||||
FROM python:3.13.15-slim
|
||||
|
||||
WORKDIR /tests
|
||||
COPY requirements.txt requirements-dev.txt ./
|
||||
RUN python -m pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt \
|
||||
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin tests
|
||||
|
||||
COPY lab.json .
|
||||
COPY app ./app
|
||||
COPY contracts ./contracts
|
||||
COPY tests ./tests
|
||||
RUN chown -R tests:tests /tests
|
||||
|
||||
USER 10001:10001
|
||||
CMD ["python", "-m", "pytest", "-q"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test test-attachments load load-attachments test-persistence test-failover test-attachment-failover reconcile-dry reconcile-apply benchmark-reconciler
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -18,9 +18,15 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachments:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) -m pytest -q tests/test_attachments.py
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
load-attachments:
|
||||
$(PYTHON) tests/upload_load.py --base-url "$(BASE_URL)" --concurrency 10 --uploads 30 $(TLS_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
@@ -28,3 +34,16 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachment-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/attachment_failover.py --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
reconcile-dry:
|
||||
docker compose exec -T api python -m app.tools.reconcile_attachments --dry-run
|
||||
|
||||
reconcile-apply:
|
||||
docker compose exec -T api python -m app.tools.reconcile_attachments --apply
|
||||
|
||||
benchmark-reconciler:
|
||||
./scripts/benchmark-reconciler.sh
|
||||
|
||||
@@ -1,37 +1,272 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Сергей Возжин, БПИ2301.
|
||||
- Целевая оценка: 5; выбран трек 2 — согласованность PostgreSQL и S3.
|
||||
- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/70
|
||||
- Использованные источники: контракт репозитория, документация Docker Compose, Chainguard Images, MinIO SDK и Pillow.
|
||||
- Библиотеки: FastAPI, Uvicorn, SQLAlchemy, asyncpg, Alembic, redis-py, MinIO Python SDK, Pillow, Argon2, pytest.
|
||||
- Помощь ИИ: OpenAI Codex использовался для плана, реализации, тестов и оформления отчёта; результат проверен исполняемыми тестами.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Engine с Compose v2, Python 3.9+ для публичных тестов и свободный host-порт 8080. Проверенный стенд: Apple M4 Pro, 12 CPU cores, 24 GB RAM, Docker Desktop; API на Python 3.13.15.
|
||||
|
||||
Закреплённые компоненты:
|
||||
|
||||
- `python:3.13.15-slim`;
|
||||
- `postgres:17.4-alpine`;
|
||||
- `redis:7.4.2-alpine`;
|
||||
- `haproxy:3.2.26-alpine`;
|
||||
- Chainguard MinIO `RELEASE.2026-10-23T20-06-51Z`, OCI index digest `sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597`;
|
||||
- Chainguard MinIO Client, вариант `latest-dev`, OCI index digest `sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805` (`mc --version` сообщает upstream build marker `DEVELOPMENT.GOGET`).
|
||||
|
||||
Готовые образы MinIO и `mc` скачиваются из реестра Chainguard `cgr.dev`; в Compose оба образа закреплены по OCI digest. Сервис `s3` использует минимальный образ `minio` и в учебном стенде запускается с `user: "0"`, чтобы иметь доступ к Docker volume `/data`; его порты на host не публикуются. Для одноразового `s3-bootstrap` используется образ `minio-client` с shell и `sed`, необходимыми для создания bucket, отдельного пользователя API и bucket-scoped policy.
|
||||
|
||||
Создать локальную конфигурацию и заменить синтетические пароли, не добавляя `.env` в Git:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
`S3_BUCKET` допускает изменение: bootstrap при каждом запуске заново применяет ограниченную
|
||||
policy с текущим именем bucket. `MINIO_ROOT_*` получает только MinIO и bootstrap; API
|
||||
получает отдельные `S3_ACCESS_KEY_ID` и `S3_SECRET_ACCESS_KEY`. Access key и secret
|
||||
считаются параметрами первоначальной инициализации пользователя MinIO; их ротация требует
|
||||
явной операторской процедуры, а не изменения `.env` на работающем volume.
|
||||
|
||||
Проверить конфигурацию, собрать образы, применить миграции и дождаться готовности:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
docker compose --env-file .env ps
|
||||
```
|
||||
|
||||
`migrate` выполняет `alembic upgrade head` отдельно и однократно. `s3-bootstrap` идемпотентно создаёт приватный bucket, отдельного пользователя API и bucket-scoped policy. Две API-реплики стартуют только после миграции, Redis и успешного bootstrap. Единственная публичная точка входа — `http://localhost:8080`; API, PostgreSQL, Redis, S3 API и административная консоль MinIO наружу не публикуются.
|
||||
|
||||
Основные проверки:
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make check
|
||||
make validate PYTHON=.venv/bin/python
|
||||
make test
|
||||
make test-attachments PYTHON=.venv/bin/python
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 api lb
|
||||
make test-attachment-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 api lb
|
||||
scripts/check-private-bucket.sh
|
||||
scripts/check-s3-outage.sh
|
||||
.venv/bin/python scripts/measure-upload-memory.py --concurrency 10 --uploads 30
|
||||
make reconcile-dry
|
||||
make benchmark-reconciler
|
||||
```
|
||||
|
||||
Исправления следует сначала просмотреть через `make reconcile-dry`. Изменяющий данные
|
||||
`make reconcile-apply` запускается оператором только после проверки dry-run.
|
||||
`make benchmark-reconciler` по умолчанию создаёт 100, 1000 и 10000 изолированных пар
|
||||
metadata/object, выполняет по три dry-run для каждого размера и удаляет тестовые данные.
|
||||
Набор и число повторов меняются через `SIZES="..."` и `RUNS=...`; старый одиночный
|
||||
параметр `SIZE=...` также поддерживается.
|
||||
|
||||
Каждый failover-тест намеренно оставляет выбранную API-реплику остановленной, поэтому между сценариями и после последнего сценария она явно восстанавливается указанной командой `docker compose up`.
|
||||
|
||||
Безопасная остановка с сохранением данных:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down
|
||||
```
|
||||
|
||||
Намеренное необратимое удаление учебных данных выполняет только оператор:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down -v
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
HTTP client
|
||||
|
|
||||
v
|
||||
HAProxy :8080
|
||||
|
|
||||
+-- round-robin --> API replica 1 --+--> PostgreSQL: metadata
|
||||
| +--> Redis: sessions
|
||||
+-- round-robin --> API replica 2 --+--> private MinIO: original bytes
|
||||
```
|
||||
|
||||
Из lab3 сохранены две взаимозаменяемые API-реплики, HAProxy, общие PostgreSQL/Redis, Bearer-сессии, `X-Instance-Id`, общая идемпотентность сообщений, readiness и failover hooks.
|
||||
|
||||
В lab4 добавлены:
|
||||
|
||||
- MinIO и persistent `s3_data` volume;
|
||||
- таблицы `attachments` и `message_attachments`;
|
||||
- синхронный binary upload PNG/JPEG;
|
||||
- авторизованные metadata/download endpoints;
|
||||
- `attachment_ids` в сообщениях и в области сравнения идемпотентного body;
|
||||
- компенсационное удаление объекта и безопасный cleanup старых orphan-объектов;
|
||||
- повторяемый reconciler метаданных PostgreSQL и оригиналов S3.
|
||||
|
||||
PostgreSQL является источником истины для метаданных. В БД не хранятся bytes изображения. MinIO хранит оригинал по серверному ключу `originals/<UUID>`. Пользовательский `filename` хранится только как метаданные и не используется в object key или пути файловой системы.
|
||||
|
||||
### Upload и лимиты
|
||||
|
||||
API сначала проверяет сессию, доступ к чату, filename и заявленный MIME. Тело читается чанками в `SpooledTemporaryFile`: до 1 MiB находится в памяти, затем временно сбрасывается на ephemeral disk API. Как только суммарный размер становится больше 10 MiB, запрос отклоняется с 413; превысивший лимит чанк в spool не записывается. Постоянной локальной копии нет.
|
||||
|
||||
Pillow определяет реальный PNG/JPEG, проверяет совпадение MIME, размеры до декодирования пикселей и целостность изображения через `verify()`. Максимум — 10 485 760 bytes и 40 000 000 пикселей. Пустой, повреждённый или чужой формат возвращает 415; превышение лимита bytes/pixels — 413.
|
||||
|
||||
S3-вызовы выполняются вне event loop. HTTP client S3 имеет ограниченные connect/read timeout и retry. Readiness проверяет PostgreSQL, Redis и существование bucket.
|
||||
|
||||
### Транзакционная граница PostgreSQL/S3
|
||||
|
||||
Атомарной транзакции между S3 и PostgreSQL нет. Для синхронного `201 ready` выбран порядок:
|
||||
|
||||
1. Полностью проверить ограниченный файл.
|
||||
2. Выполнить S3 PUT.
|
||||
3. Выполнить транзакцию INSERT метаданных.
|
||||
4. Вернуть `201` только после успешного завершения обоих шагов.
|
||||
|
||||
Если PUT не удался, метаданные не создаются. Если PUT прошёл, а транзакция БД упала, API выполняет best-effort S3 DELETE. Unit-тест с внедрённой ошибкой БД подтверждает компенсацию.
|
||||
|
||||
Если компенсационное удаление также не удалось, остаётся orphan без метаданных. CLI находит такие объекты:
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env run --rm --no-deps api \
|
||||
python -m app.tools.cleanup_orphans --dry-run --grace-seconds 3600
|
||||
|
||||
docker compose --env-file .env run --rm --no-deps api \
|
||||
python -m app.tools.cleanup_orphans --apply --grace-seconds 3600
|
||||
```
|
||||
|
||||
По умолчанию используется grace period один час; CLI запрещает значения меньше 60 секунд. Объекты с существующими метаданными и свежие orphan-объекты не удаляются. Сначала используется `--dry-run`, затем осознанный `--apply`.
|
||||
|
||||
### Reconciler PostgreSQL/S3 — трек 2
|
||||
|
||||
Гипотеза эксперимента: полный снимок PostgreSQL и listing MinIO дадут линейную
|
||||
асимптотику `O(N)` по времени и памяти. На диапазоне 100–10000 согласованных пар рост
|
||||
времени не должен становиться сверхлинейным; абсолютные значения зависят от Docker-host
|
||||
и прогрева, поэтому для каждой точки используются три последовательных запуска и медиана.
|
||||
|
||||
Расширенный reconciler запускается внутри API-контейнера, где уже есть те же настройки и
|
||||
ограниченные S3 credentials, что у приложения:
|
||||
|
||||
```sh
|
||||
docker compose exec -T api \
|
||||
python -m app.tools.reconcile_attachments --dry-run --grace-seconds 3600
|
||||
docker compose exec -T api \
|
||||
python -m app.tools.reconcile_attachments --apply --grace-seconds 3600
|
||||
```
|
||||
|
||||
Он читает снимок всех `attachments` и список `originals/`, после чего различает:
|
||||
|
||||
- `orphan`: объект S3 есть, но ни одной строки с таким `object_key` нет;
|
||||
- `missing`: строка в состоянии `ready` есть, а оригинала S3 нет;
|
||||
- корректную пару: присутствуют обе стороны;
|
||||
- свежее расхождение: возраст меньше grace period, изменять его нельзя.
|
||||
|
||||
В `--dry-run` выводятся `WOULD_DELETE_ORPHAN` и `WOULD_MARK_MISSING`, но записи и объекты
|
||||
не меняются. В `--apply` старый orphan удаляется, а missing-метаданные сохраняются и
|
||||
переходят в `status=failed`, `error_code=original_missing`. Сообщение и связь с вложением
|
||||
не удаляются. Повторный `--apply` ничего не делает: orphan уже отсутствует, а `failed`
|
||||
больше не считается ожидающим оригинал состоянием. При этом строка `failed` по-прежнему
|
||||
защищает внезапно появившийся объект с тем же ключом от удаления как orphan.
|
||||
|
||||
Grace period защищает upload, который ещё выполняется, и запрещён ниже 60 секунд.
|
||||
Используемые сообщением вложения не имеют особого исключения: любой известный PostgreSQL
|
||||
ключ защищён одинаково, поэтому корректный оригинал связанного сообщения не удаляется.
|
||||
Content для `failed` возвращает 409, а привязать такое вложение к новому сообщению нельзя.
|
||||
|
||||
Начальные снимки могут устареть во время apply. Поэтому перед удалением orphan reconciler
|
||||
повторно проверяет отсутствие `object_key` в PostgreSQL, перед маркировкой missing выполняет
|
||||
`HEAD/stat` объекта, а SQL `UPDATE` дополнительно сравнивает `id`, `object_key`, `status` и
|
||||
исходный `updated_at`. Гонки логируются как `SKIP_*` и учитываются в `race_protected`.
|
||||
Общей транзакции между PostgreSQL и S3 всё равно нет: остаётся узкое TOCTOU-окно между
|
||||
последней проверкой и действием. Полностью закрыть его можно только протоколом состояний/
|
||||
lease или запретом параллельного восстановления на время apply.
|
||||
|
||||
Операции между двумя хранилищами всё ещё не образуют общей транзакции. Если apply
|
||||
прервётся, уже выполненное безопасное удаление orphan не требует отката, условный SQL
|
||||
UPDATE меняет только всё ещё `ready` строки, а повторный запуск завершает оставшуюся
|
||||
работу.
|
||||
|
||||
### Доступ и идемпотентность
|
||||
|
||||
Метаданные и content доступны любому участнику чата. Неизвестный объект и запрос постороннего одинаково возвращают 404. Download идёт через API без redirect. Публичный ответ не раскрывает object key, endpoint или credentials.
|
||||
|
||||
Сообщение принимает 0–10 уникальных `attachment_ids`. Все вложения должны быть `ready` и относиться к тому же чату. Пустой текст допустим только с вложением. Сравнение идемпотентного запроса включает упорядоченный список attachment ID; отсутствие поля и `[]` считаются одинаковыми.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Offline/OpenAPI | `make check`; `make validate PYTHON=.venv/bin/python` | оба PASS | `evidence/results.md` |
|
||||
| Публичный smoke | `make test` | `PASS: lab4 public API scenario` | `evidence/results.md` |
|
||||
| Негативные сценарии | `make test-attachments PYTHON=.venv/bin/python` | `12 passed` | `tests/test_attachments.py`, `evidence/results.md` |
|
||||
| Компенсация S3/БД | unit-тест с ошибкой repository | S3 DELETE вызван для загруженного ключа; `1 passed` | `tests/test_attachment_compensation.py` |
|
||||
| Private bucket | `scripts/check-private-bucket.sh` | anonymous GET получил 403 | `evidence/results.md` |
|
||||
| Недоступность S3 | `scripts/check-s3-outage.sh` | readiness вернул структурированный 503; стенд восстановлен | `evidence/results.md` |
|
||||
| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | прежние token, metadata и bytes доступны после restart PostgreSQL/Redis/MinIO/API | `evidence/results.md` |
|
||||
| Общий lab3 failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | 111 записей, 1 переходная ошибка, восстановление 2,333 с, файл сохранён | `evidence/results.md` |
|
||||
| Точный attachment failover | `make test-attachment-failover ACTION_SCRIPT=scripts/stop-one.sh` | загрузившая реплика остановлена, download через другую за 3,912 с, SHA-256 совпал | `evidence/results.md` |
|
||||
| Concurrency/RAM | `scripts/measure-upload-memory.py --concurrency 10 --uploads 30` | 30/30 upload, обе реплики, пики 68,50/68,33 MiB | `evidence/upload-memory.json` |
|
||||
| Идемпотентный bootstrap | повторный запуск `docker compose up` после restart/failover | bootstrap повторно завершился с code 0 | `evidence/results.md` |
|
||||
| Cleanup dry-run | `cleanup_orphans --dry-run --grace-seconds 60` | 164 известных объекта, удалено 0 | `evidence/results.md` |
|
||||
| Reconciler fault injection | реальный PostgreSQL + MinIO: orphan, missing, fresh и три конкурентных изменения | apply исправляет только подтверждённые расхождения; свежие/изменённые данные защищены; повторный apply — 0 действий | `tests/test_reconciler_integration.py`, `evidence/results.md` |
|
||||
| Полный тестовый стенд | изолированный Compose, все pytest | `17 passed` | `evidence/results.md` |
|
||||
| Стоимость обхода | `SIZES="100 1000 10000" RUNS=3 make benchmark-reconciler` | медианы: 80,712 / 227,467 / 1733,545 мс; Python peak 2,62 / 4,21 / 8,08 MB | `evidence/results.md` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Проверяемые ожидания:
|
||||
|
||||
- остановка загрузившей API-реплики не меняет файл и не лишает вторую реплику доступа;
|
||||
- restart обязательных хранилищ с volumes сохраняет метаданные, bytes и сессию;
|
||||
- недоступность S3 наблюдаема как 503, а не 500;
|
||||
- 10 параллельных upload не приводят к неограниченному росту RAM.
|
||||
|
||||
Профиль upload: 30 запросов, concurrency 10, fixture 111 817 bytes, closed-loop, без retry. Получено 30 успешных upload за 0,382 с; максимальная latency 0,300 с; задействованы обе API-реплики. Пиковая наблюдаемая RAM: 68,50 MiB и 68,33 MiB при лимите 256 MiB на реплику.
|
||||
|
||||
Fault injection reconciler выполняется на настоящих PostgreSQL и MinIO. Тест создаёт
|
||||
старую `ready` строку без объекта, объект без строки и корректное вложение, уже связанное
|
||||
с сообщением. Дополнительно он восстанавливает missing-объект, изменяет `updated_at`
|
||||
другой missing-строки и создаёт metadata для orphan уже после получения начальных
|
||||
снимков, а также проверяет настоящий свежий orphan.
|
||||
Управляемые часы имитируют истечение grace period без минутного ожидания. Apply исправляет
|
||||
только подтверждённые расхождения; свежий объект и все три конкурентно изменённые
|
||||
сущности защищены. Повторный apply не повторяет исправления, связь сообщения сохраняется.
|
||||
|
||||
Воспроизводимый benchmark для каждого размера удаляет прежнюю фикстуру, создаёт одинаковое
|
||||
число строк `attachments` и объектов MinIO, выполняет три полных dry-run и удаляет данные.
|
||||
|
||||
| Пар metadata/object | Время, мс (три запуска) | Медиана, мс | Медианный peak Python allocations |
|
||||
| ---: | --- | ---: | ---: |
|
||||
| 100 | 78,847; 80,712; 82,781 | 80,712 | 2 620 657 bytes |
|
||||
| 1 000 | 232,254; 227,467; 226,264 | 227,467 | 4 205 016 bytes |
|
||||
| 10 000 | 1701,005; 1792,955; 1733,545 | 1733,545 | 8 075 853 bytes |
|
||||
|
||||
Измерение включает запрос PostgreSQL, listing MinIO и сравнение снимков; подготовка и
|
||||
очистка в `elapsed_ms` не входят. Результат согласуется с гипотезой `O(N)`: после заметной
|
||||
фиксированной стоимости время и память монотонно растут, на исследованном диапазоне нет
|
||||
признаков сверхлинейного взрыва. Это не доказательство для production-масштаба: измеряется
|
||||
Python `tracemalloc`, а не RSS контейнера, и полный снимок всё равно требует памяти `O(N)`.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Restart API + PostgreSQL + Redis + MinIO | token, metadata и bytes сохраняются | persistence PASS | потерь нет |
|
||||
| Остановка API в общем failover | вторая реплика продолжает чтение/запись | 111 записей, 1 краткая ошибка | 2,333 с; потерь и дублей нет |
|
||||
| Остановка загрузившей API | файл скачивается через другую | SHA-256 совпал, разные instance ID | 3,912 с; потерь нет |
|
||||
| Остановка MinIO | readiness сообщает отказ зависимости | структурированный 503, после запуска readiness восстановился | потерь нет |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Стенд выдерживает остановку одной API-реплики и штатный restart контейнеров с сохранёнными volumes. Он не выдерживает потерю Docker-host, единственного HAProxy, PostgreSQL, Redis или MinIO volume. Несколько API-процессов не делают единственные хранилища высокодоступными.
|
||||
|
||||
Upload синхронный, поэтому клиент ждёт S3 и PostgreSQL. `SpooledTemporaryFile` ограничивает RAM одной загрузки, но временный disk API должен иметь достаточный объём и отдельную квоту в production. Reconciler загружает полный список объектов и строк БД в память, поэтому его стоимость O(N), а для большого bucket нужны страничный/потоковый обход, метрики, checkpoint и плановый запуск.
|
||||
|
||||
Chainguard-образы зафиксированы по digest, поэтому обновляются только явно. Стенд изолирован учебной сетью. Перед production-развёртыванием потребовались бы непривилегированный UID с корректными правами на volume, TLS, внешний secret manager, HA хранилищ, аудит и резервное копирование.
|
||||
|
||||
Presigned upload, worker, thumbnail и варианты изображений намеренно не реализованы: для выбранного трека 2 они не требуются.
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
sqlalchemy.url = placeholder://configured-at-runtime
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
|
||||
from app.config import load_settings
|
||||
from app.db.postgres import Base
|
||||
from app import models # noqa: F401 - imports ORM models into Base.metadata
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def _url() -> str:
|
||||
return load_settings().database_url
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(url=_url(), target_metadata=target_metadata, literal_binds=True, dialect_opts={'paramstyle': 'named'})
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def _run_migrations(connection) -> None:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def _run_migrations_async() -> None:
|
||||
configuration = config.get_section(config.config_ini_section, {})
|
||||
configuration['sqlalchemy.url'] = _url()
|
||||
connectable = async_engine_from_config(configuration, prefix='sqlalchemy.', poolclass=pool.NullPool)
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(_run_migrations)
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
asyncio.run(_run_migrations_async())
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,22 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Initial PostgreSQL schema for lab 2.
|
||||
|
||||
Revision ID: 20261019_0001
|
||||
Revises:
|
||||
Create Date: 2026-10-19
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '20261019_0001'
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
'users',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('username', sa.String(length=32), nullable=False),
|
||||
sa.Column('display_name', sa.String(length=100), nullable=False),
|
||||
sa.Column('password_hash', sa.Text(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('username'),
|
||||
)
|
||||
op.create_table(
|
||||
'chats',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('title', sa.String(length=100), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
)
|
||||
op.create_table(
|
||||
'chat_members',
|
||||
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('user_id', sa.Uuid(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('chat_id', 'user_id'),
|
||||
)
|
||||
op.create_index('chat_members_user_chat_idx', 'chat_members', ['user_id', 'chat_id'])
|
||||
op.create_table(
|
||||
'messages',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('sender_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('text', sa.String(length=4000), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['sender_id'], ['users.id']),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
)
|
||||
op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id'])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index('messages_history_idx', table_name='messages')
|
||||
op.drop_table('messages')
|
||||
op.drop_index('chat_members_user_chat_idx', table_name='chat_members')
|
||||
op.drop_table('chat_members')
|
||||
op.drop_table('chats')
|
||||
op.drop_table('users')
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Add shared message idempotency for lab 3.
|
||||
|
||||
Revision ID: 20261023_0002
|
||||
Revises: 20261019_0001
|
||||
Create Date: 2026-10-23
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '20261023_0002'
|
||||
down_revision = '20261019_0001'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
'messages',
|
||||
sa.Column('idempotency_key', sa.String(length=128), nullable=True),
|
||||
)
|
||||
op.create_unique_constraint(
|
||||
'messages_sender_chat_idempotency_key',
|
||||
'messages',
|
||||
['sender_id', 'chat_id', 'idempotency_key'],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint(
|
||||
'messages_sender_chat_idempotency_key',
|
||||
'messages',
|
||||
type_='unique',
|
||||
)
|
||||
op.drop_column('messages', 'idempotency_key')
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Add S3 attachment metadata and message links for lab 4.
|
||||
|
||||
Revision ID: 20261030_0003
|
||||
Revises: 20261023_0002
|
||||
Create Date: 2026-10-30
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '20261030_0003'
|
||||
down_revision = '20261023_0002'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
'attachments',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('owner_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('filename', sa.String(length=255), nullable=False),
|
||||
sa.Column('object_key', sa.String(length=255), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=32), nullable=False),
|
||||
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('width', sa.Integer(), nullable=False),
|
||||
sa.Column('height', sa.Integer(), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), nullable=False),
|
||||
sa.Column('error_code', sa.String(length=64), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['owner_id'], ['users.id']),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('object_key', name='attachments_object_key_key'),
|
||||
)
|
||||
op.create_index(
|
||||
'attachments_chat_created_idx',
|
||||
'attachments',
|
||||
['chat_id', 'created_at', 'id'],
|
||||
)
|
||||
op.create_table(
|
||||
'message_attachments',
|
||||
sa.Column('message_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('attachment_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('position', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('message_id', 'attachment_id'),
|
||||
sa.UniqueConstraint(
|
||||
'message_id',
|
||||
'position',
|
||||
name='message_attachments_message_position_key',
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
'message_attachments_attachment_idx',
|
||||
'message_attachments',
|
||||
['attachment_id'],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index('message_attachments_attachment_idx', table_name='message_attachments')
|
||||
op.drop_table('message_attachments')
|
||||
op.drop_index('attachments_chat_created_idx', table_name='attachments')
|
||||
op.drop_table('attachments')
|
||||
@@ -0,0 +1 @@
|
||||
"""Мессенджер на PostgreSQL и Redis, лабораторная 2."""
|
||||
@@ -0,0 +1 @@
|
||||
"""HTTP-граница: FastAPI, коды состояния, формат ошибок, курсоры."""
|
||||
@@ -0,0 +1,28 @@
|
||||
import base64
|
||||
import json
|
||||
from uuid import UUID
|
||||
|
||||
from ..domain.errors import Invalid
|
||||
from ..domain.models import Keyset
|
||||
from .timefmt import from_rfc3339, to_rfc3339
|
||||
|
||||
MAX_LENGTH = 512
|
||||
|
||||
|
||||
def encode(scope: str, position: Keyset) -> str:
|
||||
payload = {'s': scope, 't': to_rfc3339(position.created_at), 'i': str(position.id)}
|
||||
raw = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8')
|
||||
return base64.urlsafe_b64encode(raw).decode('ascii').rstrip('=')
|
||||
|
||||
|
||||
def decode(scope: str, value: str) -> Keyset:
|
||||
if not value or len(value) > MAX_LENGTH:
|
||||
raise Invalid('cursor недействителен для этого запроса')
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4))
|
||||
payload = json.loads(raw)
|
||||
if not isinstance(payload, dict) or payload.get('s') != scope:
|
||||
raise ValueError('scope mismatch')
|
||||
return Keyset(created_at=from_rfc3339(payload['t']), id=UUID(payload['i']))
|
||||
except Exception:
|
||||
raise Invalid('cursor недействителен для этого запроса') from None
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from fastapi import Depends, Header, HTTPException, Request, status
|
||||
|
||||
from ..db.redis import RedisSessionStore
|
||||
from ..domain.errors import Unauthorized
|
||||
from ..domain.models import User
|
||||
from ..services.chats import ChatService
|
||||
from ..services.attachments import AttachmentService
|
||||
from ..services.messages import MessageService
|
||||
from ..services.users import UserService
|
||||
|
||||
_BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Services:
|
||||
attachments: AttachmentService
|
||||
users: UserService
|
||||
chats: ChatService
|
||||
messages: MessageService
|
||||
sessions: RedisSessionStore
|
||||
|
||||
|
||||
def services(request: Request) -> Services:
|
||||
return request.app.state.services
|
||||
|
||||
|
||||
def user_service(container: Services = Depends(services)) -> UserService:
|
||||
return container.users
|
||||
|
||||
|
||||
def chat_service(container: Services = Depends(services)) -> ChatService:
|
||||
return container.chats
|
||||
|
||||
|
||||
def message_service(container: Services = Depends(services)) -> MessageService:
|
||||
return container.messages
|
||||
|
||||
|
||||
def attachment_service(container: Services = Depends(services)) -> AttachmentService:
|
||||
return container.attachments
|
||||
|
||||
|
||||
def session_store(container: Services = Depends(services)) -> RedisSessionStore:
|
||||
return container.sessions
|
||||
|
||||
|
||||
def _bearer_error() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail='требуется действующая Bearer-сессия',
|
||||
headers={'WWW-Authenticate': _BEARER_CHALLENGE},
|
||||
)
|
||||
|
||||
|
||||
async def bearer_token(authorization: str | None = Header(None)) -> str:
|
||||
if not authorization:
|
||||
raise _bearer_error()
|
||||
scheme, separator, token = authorization.partition(' ')
|
||||
if scheme.lower() != 'bearer' or not separator or not token or token.strip() != token:
|
||||
raise _bearer_error()
|
||||
return token
|
||||
|
||||
|
||||
async def current_user(
|
||||
token: str = Depends(bearer_token),
|
||||
container: Services = Depends(services),
|
||||
) -> User:
|
||||
user_id = await container.sessions.get_user_id(token)
|
||||
if user_id is None:
|
||||
raise _bearer_error()
|
||||
try:
|
||||
return await container.users.identify(user_id)
|
||||
except Unauthorized:
|
||||
raise _bearer_error() from None
|
||||
@@ -0,0 +1,107 @@
|
||||
import logging
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from ..domain.errors import (
|
||||
Conflict,
|
||||
DomainError,
|
||||
Invalid,
|
||||
NotFound,
|
||||
PayloadTooLarge,
|
||||
Unauthorized,
|
||||
Unavailable,
|
||||
UnsupportedMediaType,
|
||||
)
|
||||
|
||||
logger = logging.getLogger('app.error')
|
||||
|
||||
CODE_BY_STATUS = {
|
||||
400: 'invalid_request',
|
||||
401: 'unauthorized',
|
||||
404: 'not_found',
|
||||
409: 'conflict',
|
||||
413: 'payload_too_large',
|
||||
415: 'unsupported_media_type',
|
||||
429: 'rate_limited',
|
||||
500: 'internal_error',
|
||||
503: 'unavailable',
|
||||
}
|
||||
|
||||
STATUS_BY_DOMAIN = {
|
||||
Invalid: 400,
|
||||
Unauthorized: 401,
|
||||
NotFound: 404,
|
||||
Conflict: 409,
|
||||
PayloadTooLarge: 413,
|
||||
UnsupportedMediaType: 415,
|
||||
Unavailable: 503,
|
||||
}
|
||||
|
||||
|
||||
def code_for(status: int) -> str:
|
||||
if status in CODE_BY_STATUS:
|
||||
return CODE_BY_STATUS[status]
|
||||
return 'invalid_request' if status < 500 else 'internal_error'
|
||||
|
||||
|
||||
def error_response(status: int, message: str, request_id: str, code: str = '') -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={'error': {'code': code or code_for(status), 'message': message, 'request_id': request_id}},
|
||||
)
|
||||
|
||||
|
||||
def _domain_status(error: DomainError) -> int:
|
||||
for kind, status in STATUS_BY_DOMAIN.items():
|
||||
if isinstance(error, kind):
|
||||
return status
|
||||
return 500
|
||||
|
||||
|
||||
def _describe(error: RequestValidationError) -> str:
|
||||
"""Краткое указание на пол, без stack trace и внутреннего устройства"""
|
||||
for item in error.errors():
|
||||
if item.get('type') == 'json_invalid':
|
||||
return 'тело запроса не является корректным JSON'
|
||||
location = '.'.join(str(part) for part in item.get('loc', ()) if part != 'body')
|
||||
return f'некорректное поле {location}' if location else 'некорректный запрос'
|
||||
return 'некорректный запрос'
|
||||
|
||||
|
||||
def request_id(request) -> str:
|
||||
return getattr(request.state, 'request_id', 'unknown')
|
||||
|
||||
|
||||
def install(app: FastAPI) -> None:
|
||||
@app.exception_handler(DomainError)
|
||||
async def _on_domain(request, error: DomainError) -> JSONResponse:
|
||||
return error_response(_domain_status(error), error.message, request_id(request))
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _on_validation(request, error: RequestValidationError) -> JSONResponse:
|
||||
return error_response(400, _describe(error), request_id(request))
|
||||
|
||||
@app.exception_handler(SQLAlchemyError)
|
||||
async def _on_database(request, error: SQLAlchemyError) -> JSONResponse:
|
||||
logger.warning('database error request_id=%s type=%s', request_id(request), type(error).__name__)
|
||||
return error_response(503, 'база данных недоступна', request_id(request))
|
||||
|
||||
@app.exception_handler(OSError)
|
||||
async def _on_storage_socket(request, error: OSError) -> JSONResponse:
|
||||
logger.warning('storage socket error request_id=%s type=%s', request_id(request), type(error).__name__)
|
||||
return error_response(503, 'обязательное хранилище недоступно', request_id(request))
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def _on_http(request, error: StarletteHTTPException) -> JSONResponse:
|
||||
message = error.detail if isinstance(error.detail, str) and error.detail else 'ошибка запроса'
|
||||
response = error_response(error.status_code, message, request_id(request))
|
||||
for name, value in (error.headers or {}).items():
|
||||
response.headers[name] = value
|
||||
return response
|
||||
|
||||
|
||||
__all__ = ['code_for', 'error_response', 'install']
|
||||
@@ -0,0 +1,43 @@
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
from uuid import uuid4
|
||||
|
||||
from starlette.datastructures import MutableHeaders
|
||||
|
||||
from .errors import error_response
|
||||
|
||||
logger = logging.getLogger('app')
|
||||
|
||||
|
||||
class RequestContext:
|
||||
def __init__(self, app) -> None:
|
||||
self.app = app
|
||||
self.instance_id = os.getenv('INSTANCE_ID') or socket.gethostname()
|
||||
|
||||
async def __call__(self, scope, receive, send) -> None:
|
||||
if scope['type'] != 'http':
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
request_id = uuid4().hex
|
||||
scope.setdefault('state', {})['request_id'] = request_id
|
||||
started = False
|
||||
|
||||
async def send_wrapper(message) -> None:
|
||||
nonlocal started
|
||||
if message['type'] == 'http.response.start':
|
||||
started = True
|
||||
headers = MutableHeaders(scope=message)
|
||||
headers['X-Request-Id'] = request_id
|
||||
headers['X-Instance-Id'] = self.instance_id
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, send_wrapper)
|
||||
except Exception:
|
||||
logger.exception(f'unhandled error request_id={request_id} path={scope.get("path")}')
|
||||
if started:
|
||||
raise
|
||||
response = error_response(500, 'внутренняя ошибка', request_id)
|
||||
await response(scope, receive, send_wrapper)
|
||||
@@ -0,0 +1,3 @@
|
||||
from . import auth, chats, health, messages, users
|
||||
|
||||
__all__ = ['auth', 'chats', 'health', 'messages', 'users']
|
||||
@@ -0,0 +1,66 @@
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from ...domain.models import User
|
||||
from ...services.attachments import AttachmentService
|
||||
from ..deps import attachment_service, current_user
|
||||
from ..schemas import AttachmentOut
|
||||
|
||||
router = APIRouter(tags=['attachments'])
|
||||
|
||||
|
||||
@router.post(
|
||||
'/api/v1/chats/{chat_id}/attachments',
|
||||
status_code=201,
|
||||
response_model=AttachmentOut,
|
||||
)
|
||||
async def upload(
|
||||
chat_id: UUID,
|
||||
request: Request,
|
||||
filename: str = Query(..., min_length=1, max_length=255),
|
||||
content_type: Annotated[str | None, Header(alias='Content-Type')] = None,
|
||||
actor: User = Depends(current_user),
|
||||
attachments: AttachmentService = Depends(attachment_service),
|
||||
) -> AttachmentOut:
|
||||
attachment = await attachments.upload(
|
||||
actor,
|
||||
chat_id,
|
||||
filename,
|
||||
content_type or '',
|
||||
request.stream(),
|
||||
)
|
||||
return AttachmentOut.from_domain(attachment)
|
||||
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}', response_model=AttachmentOut)
|
||||
async def metadata(
|
||||
attachment_id: UUID,
|
||||
actor: User = Depends(current_user),
|
||||
attachments: AttachmentService = Depends(attachment_service),
|
||||
) -> AttachmentOut:
|
||||
return AttachmentOut.from_domain(await attachments.get(actor, attachment_id))
|
||||
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}/content')
|
||||
async def content(
|
||||
attachment_id: UUID,
|
||||
variant: str = Query('original'),
|
||||
actor: User = Depends(current_user),
|
||||
attachments: AttachmentService = Depends(attachment_service),
|
||||
) -> StreamingResponse:
|
||||
if variant != 'original':
|
||||
from ...domain.errors import NotFound
|
||||
|
||||
raise NotFound('вариант вложения не найден')
|
||||
attachment, stream = await attachments.content(actor, attachment_id)
|
||||
return StreamingResponse(
|
||||
stream.chunks(),
|
||||
media_type=attachment.content_type,
|
||||
headers={
|
||||
'Content-Length': str(attachment.size_bytes),
|
||||
'Content-Disposition': f'inline; filename="attachment-{attachment.id}"',
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from binascii import Error as Base64Error
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, status
|
||||
|
||||
from ...auth.tokens import create_token
|
||||
from ...db.redis import RedisSessionStore
|
||||
from ...domain.models import User
|
||||
from ...services.users import UserService
|
||||
from ..deps import bearer_token, current_user, session_store, user_service
|
||||
from ..schemas import SessionOut, UserOut
|
||||
from ..timefmt import to_rfc3339
|
||||
|
||||
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
|
||||
|
||||
_BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
|
||||
|
||||
def _unauthorized_basic() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail='неверные учётные данные',
|
||||
headers={'WWW-Authenticate': _BASIC_CHALLENGE},
|
||||
)
|
||||
|
||||
|
||||
def _parse_basic(authorization: str | None) -> tuple[str, str]:
|
||||
if not authorization:
|
||||
raise _unauthorized_basic()
|
||||
scheme, separator, encoded = authorization.partition(' ')
|
||||
if scheme.lower() != 'basic' or not separator or not encoded:
|
||||
raise _unauthorized_basic()
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode('utf-8')
|
||||
except (Base64Error, UnicodeDecodeError):
|
||||
raise _unauthorized_basic() from None
|
||||
username, separator, password = decoded.partition(':')
|
||||
if not separator or not username:
|
||||
raise _unauthorized_basic()
|
||||
return username, password
|
||||
|
||||
|
||||
@router.post('', status_code=status.HTTP_201_CREATED, response_model=SessionOut)
|
||||
async def create_session(
|
||||
authorization: str | None = Header(None),
|
||||
users: UserService = Depends(user_service),
|
||||
sessions: RedisSessionStore = Depends(session_store),
|
||||
) -> SessionOut:
|
||||
username, password = _parse_basic(authorization)
|
||||
user = await users.authenticate(username, password)
|
||||
if user is None:
|
||||
raise _unauthorized_basic()
|
||||
token = create_token()
|
||||
expires_at = await sessions.create(user.id, token)
|
||||
return SessionOut(
|
||||
token=token,
|
||||
token_type='Bearer',
|
||||
expires_at=to_rfc3339(expires_at),
|
||||
user=UserOut.from_domain(user),
|
||||
)
|
||||
|
||||
|
||||
@router.delete('/current', status_code=status.HTTP_204_NO_CONTENT, response_model=None)
|
||||
async def revoke_session(
|
||||
token: str = Depends(bearer_token),
|
||||
_: User = Depends(current_user),
|
||||
sessions: RedisSessionStore = Depends(session_store),
|
||||
) -> None:
|
||||
await sessions.revoke(token)
|
||||
@@ -0,0 +1,49 @@
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from ...domain.models import User
|
||||
from ...services.chats import ChatService
|
||||
from .. import cursors
|
||||
from ..deps import chat_service, current_user
|
||||
from ..schemas import (
|
||||
LIMIT_DEFAULT,
|
||||
LIMIT_MAX,
|
||||
LIMIT_MIN,
|
||||
ChatOut,
|
||||
ChatPageOut,
|
||||
CreateChat,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix='/api/v1/chats', tags=['chats'])
|
||||
|
||||
|
||||
@router.post('', status_code=201, response_model=ChatOut)
|
||||
async def create(
|
||||
body: CreateChat,
|
||||
actor: User = Depends(current_user),
|
||||
chats: ChatService = Depends(chat_service),
|
||||
) -> ChatOut:
|
||||
return ChatOut.from_domain(await chats.create(actor, body.title, body.member_ids))
|
||||
|
||||
|
||||
@router.get('', response_model=ChatPageOut)
|
||||
async def page(
|
||||
actor: User = Depends(current_user),
|
||||
chats: ChatService = Depends(chat_service),
|
||||
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
|
||||
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
|
||||
) -> ChatPageOut:
|
||||
scope = f'chats:{actor.id}'
|
||||
after = cursors.decode(scope, cursor) if cursor is not None else None
|
||||
return ChatPageOut.from_domain(await chats.page(actor, after, limit), scope)
|
||||
|
||||
|
||||
@router.get('/{chat_id}', response_model=ChatOut)
|
||||
async def get(
|
||||
chat_id: UUID,
|
||||
actor: User = Depends(current_user),
|
||||
chats: ChatService = Depends(chat_service),
|
||||
) -> ChatOut:
|
||||
return ChatOut.from_domain(await chats.get(actor, chat_id))
|
||||
@@ -0,0 +1,29 @@
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from redis.exceptions import RedisError
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from ...domain.errors import Unavailable
|
||||
|
||||
from ..schemas import HealthOut
|
||||
|
||||
router = APIRouter(tags=['health'])
|
||||
|
||||
|
||||
@router.get('/health/live', response_model=HealthOut)
|
||||
async def live() -> HealthOut:
|
||||
return HealthOut(status='ok')
|
||||
|
||||
|
||||
@router.get('/health/ready', response_model=HealthOut)
|
||||
async def ready(request: Request) -> HealthOut:
|
||||
if not request.app.state.ready:
|
||||
raise HTTPException(status_code=503, detail='приложение не готово принимать трафик')
|
||||
try:
|
||||
async with request.app.state.engine.connect() as connection:
|
||||
await connection.execute(text('SELECT 1'))
|
||||
await request.app.state.redis.ping()
|
||||
await request.app.state.storage.ping()
|
||||
except (SQLAlchemyError, RedisError, Unavailable, OSError):
|
||||
raise HTTPException(status_code=503, detail='обязательное хранилище недоступно') from None
|
||||
return HealthOut(status='ok')
|
||||
@@ -0,0 +1,53 @@
|
||||
from typing import Annotated, Optional
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, Query
|
||||
|
||||
from ...domain.models import User
|
||||
from ...services.messages import MessageService
|
||||
from .. import cursors
|
||||
from ..deps import current_user, message_service
|
||||
from ..schemas import (
|
||||
LIMIT_DEFAULT,
|
||||
LIMIT_MAX,
|
||||
LIMIT_MIN,
|
||||
CreateMessage,
|
||||
MessageOut,
|
||||
MessagePageOut,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix='/api/v1/chats/{chat_id}/messages', tags=['messages'])
|
||||
|
||||
|
||||
@router.post('', status_code=201, response_model=MessageOut)
|
||||
async def send(
|
||||
chat_id: UUID,
|
||||
body: CreateMessage,
|
||||
idempotency_key: Annotated[
|
||||
str,
|
||||
Header(
|
||||
alias='Idempotency-Key',
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r'^[A-Za-z0-9._:-]+$',
|
||||
),
|
||||
],
|
||||
actor: User = Depends(current_user),
|
||||
messages: MessageService = Depends(message_service),
|
||||
) -> MessageOut:
|
||||
return MessageOut.from_domain(
|
||||
await messages.send(actor, chat_id, body.text, body.attachment_ids, idempotency_key)
|
||||
)
|
||||
|
||||
|
||||
@router.get('', response_model=MessagePageOut)
|
||||
async def page(
|
||||
chat_id: UUID,
|
||||
actor: User = Depends(current_user),
|
||||
messages: MessageService = Depends(message_service),
|
||||
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
|
||||
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
|
||||
) -> MessagePageOut:
|
||||
scope = f'messages:{chat_id}'
|
||||
after = cursors.decode(scope, cursor) if cursor is not None else None
|
||||
return MessagePageOut.from_domain(await messages.history(actor, chat_id, after, limit), scope)
|
||||
@@ -0,0 +1,18 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from ...domain.models import User
|
||||
from ...services.users import UserService
|
||||
from ..deps import current_user, user_service
|
||||
from ..schemas import CreateUser, UserOut
|
||||
|
||||
router = APIRouter(prefix='/api/v1', tags=['users'])
|
||||
|
||||
|
||||
@router.post('/users', status_code=201, response_model=UserOut)
|
||||
async def register(body: CreateUser, users: UserService = Depends(user_service)) -> UserOut:
|
||||
return UserOut.from_domain(await users.register(body.username, body.display_name, body.password))
|
||||
|
||||
|
||||
@router.get('/users/me', response_model=UserOut)
|
||||
async def me(actor: User = Depends(current_user)) -> UserOut:
|
||||
return UserOut.from_domain(actor)
|
||||
@@ -0,0 +1,190 @@
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from ..domain.models import Attachment, Chat, Message, Page, User
|
||||
from . import cursors
|
||||
from .timefmt import to_rfc3339
|
||||
|
||||
NAME_MAX = 100
|
||||
TEXT_MAX = 4000
|
||||
MEMBERS_MAX = 100
|
||||
LIMIT_MIN = 1
|
||||
LIMIT_MAX = 100
|
||||
LIMIT_DEFAULT = 50
|
||||
|
||||
|
||||
def not_blank(value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError('строка не может состоять только из пробельных символов')
|
||||
return value
|
||||
|
||||
|
||||
class Strict(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
|
||||
|
||||
class HealthOut(Strict):
|
||||
status: str = 'ok'
|
||||
|
||||
|
||||
class CreateUser(Strict):
|
||||
username: str = Field(min_length=3, max_length=32, pattern=r'^[a-z0-9_]{3,32}$')
|
||||
display_name: str = Field(min_length=1, max_length=NAME_MAX)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator('display_name')
|
||||
@classmethod
|
||||
def _display_name(cls, value: str) -> str:
|
||||
return not_blank(value)
|
||||
|
||||
|
||||
class UserOut(Strict):
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, user: User) -> 'UserOut':
|
||||
return cls(
|
||||
id=user.id,
|
||||
username=user.username,
|
||||
display_name=user.display_name,
|
||||
created_at=to_rfc3339(user.created_at),
|
||||
)
|
||||
|
||||
|
||||
class SessionOut(Strict):
|
||||
token: str = Field(min_length=32)
|
||||
token_type: str = 'Bearer'
|
||||
expires_at: str
|
||||
user: UserOut
|
||||
|
||||
|
||||
class CreateChat(Strict):
|
||||
title: str = Field(min_length=1, max_length=NAME_MAX)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=MEMBERS_MAX)
|
||||
|
||||
@field_validator('title')
|
||||
@classmethod
|
||||
def _title(cls, value: str) -> str:
|
||||
return not_blank(value)
|
||||
|
||||
@field_validator('member_ids')
|
||||
@classmethod
|
||||
def _members_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError('member_ids не должен содержать дубликаты')
|
||||
return value
|
||||
|
||||
|
||||
class ChatOut(Strict):
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: list[UUID]
|
||||
created_at: str
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, chat: Chat) -> 'ChatOut':
|
||||
return cls(
|
||||
id=chat.id,
|
||||
title=chat.title,
|
||||
member_ids=list(chat.member_ids),
|
||||
created_at=to_rfc3339(chat.created_at),
|
||||
)
|
||||
|
||||
|
||||
class CreateMessage(Strict):
|
||||
text: str = Field(max_length=TEXT_MAX)
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator('attachment_ids')
|
||||
@classmethod
|
||||
def _attachments_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError('attachment_ids не должен содержать дубликаты')
|
||||
return value
|
||||
|
||||
@model_validator(mode='after')
|
||||
def _content_present(self) -> 'CreateMessage':
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError('сообщение должно содержать текст или вложение')
|
||||
return self
|
||||
|
||||
|
||||
class MessageOut(Strict):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: list[UUID]
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, message: Message) -> 'MessageOut':
|
||||
return cls(
|
||||
id=message.id,
|
||||
chat_id=message.chat_id,
|
||||
sender_id=message.sender_id,
|
||||
text=message.text,
|
||||
created_at=to_rfc3339(message.created_at),
|
||||
attachment_ids=list(message.attachment_ids),
|
||||
)
|
||||
|
||||
|
||||
class AttachmentOut(Strict):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str
|
||||
variants: list = Field(default_factory=list)
|
||||
error_code: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, attachment: Attachment) -> 'AttachmentOut':
|
||||
return cls(
|
||||
id=attachment.id,
|
||||
chat_id=attachment.chat_id,
|
||||
owner_id=attachment.owner_id,
|
||||
filename=attachment.filename,
|
||||
content_type=attachment.content_type,
|
||||
size_bytes=attachment.size_bytes,
|
||||
sha256=attachment.sha256,
|
||||
status=attachment.status,
|
||||
variants=[],
|
||||
error_code=attachment.error_code,
|
||||
created_at=to_rfc3339(attachment.created_at),
|
||||
updated_at=to_rfc3339(attachment.updated_at),
|
||||
)
|
||||
|
||||
|
||||
class ChatPageOut(Strict):
|
||||
items: list[ChatOut]
|
||||
next_cursor: Optional[str]
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, page: Page, scope: str) -> 'ChatPageOut':
|
||||
return cls(
|
||||
items=[ChatOut.from_domain(chat) for chat in page.items],
|
||||
next_cursor=cursors.encode(scope, page.next) if page.next else None,
|
||||
)
|
||||
|
||||
|
||||
class MessagePageOut(Strict):
|
||||
items: list[MessageOut]
|
||||
next_cursor: Optional[str]
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, page: Page, scope: str) -> 'MessagePageOut':
|
||||
return cls(
|
||||
items=[MessageOut.from_domain(message) for message in page.items],
|
||||
next_cursor=cursors.encode(scope, page.next) if page.next else None,
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def to_rfc3339(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def from_rfc3339(value: str) -> datetime:
|
||||
if not isinstance(value, str) or not value.endswith('Z'):
|
||||
raise ValueError('timestamp must end with Z')
|
||||
return datetime.fromisoformat(value[:-1]).replace(tzinfo=timezone.utc)
|
||||
@@ -0,0 +1,4 @@
|
||||
from .passwords import hash_password, password_matches
|
||||
from .tokens import create_token
|
||||
|
||||
__all__ = ['create_token', 'hash_password', 'password_matches']
|
||||
@@ -0,0 +1,15 @@
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError, VerifyMismatchError
|
||||
|
||||
_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def password_matches(stored_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return _hasher.verify(stored_hash, password)
|
||||
except (VerifyMismatchError, VerificationError):
|
||||
return False
|
||||
@@ -0,0 +1,11 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
|
||||
|
||||
def create_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def session_key(token: str) -> str:
|
||||
digest = hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
return f'session:{digest}'
|
||||
@@ -0,0 +1,3 @@
|
||||
from .settings import Settings, load_settings
|
||||
|
||||
__all__ = ['Settings', 'load_settings']
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _read_value(name: str, file_name: str) -> str:
|
||||
value = os.getenv(name)
|
||||
if value:
|
||||
return value
|
||||
path = os.getenv(file_name)
|
||||
if path:
|
||||
return Path(path).read_text(encoding='utf-8').strip()
|
||||
raise RuntimeError(f'не задана настройка {name} или {file_name}')
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Settings:
|
||||
database_url: str
|
||||
redis_url: str
|
||||
session_ttl_seconds: int
|
||||
s3_endpoint: str
|
||||
s3_region: str
|
||||
s3_bucket: str
|
||||
s3_access_key_id: str
|
||||
s3_secret_access_key: str
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
|
||||
|
||||
def _positive_int(name: str, default: str) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, default))
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f'{name} должен быть целым числом') from exc
|
||||
if value <= 0:
|
||||
raise RuntimeError(f'{name} должен быть больше нуля')
|
||||
return value
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings(
|
||||
database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'),
|
||||
redis_url=_read_value('REDIS_URL', 'CACHE_URL_FILE'),
|
||||
session_ttl_seconds=_positive_int('SESSION_TTL_SECONDS', '3600'),
|
||||
s3_endpoint=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||
s3_region=os.getenv('S3_REGION', 'us-east-1'),
|
||||
s3_bucket=os.getenv('S3_BUCKET', 'messenger-attachments'),
|
||||
s3_access_key_id=_read_value('S3_ACCESS_KEY_ID', 'S3_ACCESS_KEY_ID_FILE'),
|
||||
s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'),
|
||||
max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'),
|
||||
max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'),
|
||||
)
|
||||
@@ -0,0 +1,4 @@
|
||||
from .postgres import Base, create_engine, create_session_factory
|
||||
from .redis import RedisSessionStore, create_redis
|
||||
|
||||
__all__ = ['Base', 'RedisSessionStore', 'create_engine', 'create_redis', 'create_session_factory']
|
||||
@@ -0,0 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def create_engine(url: str) -> AsyncEngine:
|
||||
return create_async_engine(
|
||||
url,
|
||||
pool_size=5,
|
||||
max_overflow=5,
|
||||
pool_timeout=5,
|
||||
pool_pre_ping=True,
|
||||
connect_args={'server_settings': {'statement_timeout': '5000'}},
|
||||
)
|
||||
|
||||
|
||||
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(engine, expire_on_commit=False)
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from ..auth.tokens import session_key
|
||||
from ..domain.errors import Unavailable
|
||||
|
||||
|
||||
def create_redis(url: str) -> Redis:
|
||||
return Redis.from_url(url, decode_responses=True, socket_connect_timeout=3, socket_timeout=3)
|
||||
|
||||
|
||||
class RedisSessionStore:
|
||||
def __init__(self, client: Redis, ttl_seconds: int) -> None:
|
||||
self._client = client
|
||||
self._ttl_seconds = ttl_seconds
|
||||
|
||||
async def create(self, user_id: UUID, token: str) -> datetime:
|
||||
try:
|
||||
await self._client.set(session_key(token), str(user_id), ex=self._ttl_seconds)
|
||||
except RedisError as exc:
|
||||
raise Unavailable('хранилище сессий недоступно') from exc
|
||||
return datetime.now(timezone.utc) + timedelta(seconds=self._ttl_seconds)
|
||||
|
||||
async def get_user_id(self, token: str) -> UUID | None:
|
||||
try:
|
||||
value = await self._client.get(session_key(token))
|
||||
except RedisError as exc:
|
||||
raise Unavailable('хранилище сессий недоступно') from exc
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return UUID(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def revoke(self, token: str) -> None:
|
||||
try:
|
||||
await self._client.delete(session_key(token))
|
||||
except RedisError as exc:
|
||||
raise Unavailable('хранилище сессий недоступно') from exc
|
||||
|
||||
async def ping(self) -> None:
|
||||
try:
|
||||
await self._client.ping()
|
||||
except RedisError as exc:
|
||||
raise Unavailable('хранилище сессий недоступно') from exc
|
||||
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class DomainError(Exception):
|
||||
"""Сообщение попадает клиенту, поэтому не содержит внутренних деталей."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
|
||||
|
||||
class Invalid(DomainError):
|
||||
"""Запрос нарушает правила предметной области."""
|
||||
|
||||
|
||||
class Unauthorized(DomainError):
|
||||
"""Нет действительной идентификации."""
|
||||
|
||||
|
||||
class NotFound(DomainError):
|
||||
"""Объекта нет либо он недоступен вызывающему: снаружи это неразличимо."""
|
||||
|
||||
|
||||
class Conflict(DomainError):
|
||||
"""Состояние не позволяет выполнить операцию."""
|
||||
|
||||
|
||||
class Unavailable(DomainError):
|
||||
"""Обязательная внешняя зависимость временно недоступна."""
|
||||
|
||||
|
||||
class PayloadTooLarge(DomainError):
|
||||
"""Тело запроса или декодированное содержимое превышает лимит."""
|
||||
|
||||
|
||||
class UnsupportedMediaType(DomainError):
|
||||
"""Формат содержимого не поддерживается или не соответствует MIME."""
|
||||
@@ -0,0 +1,75 @@
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Chat:
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: tuple[UUID, ...]
|
||||
created_at: datetime
|
||||
|
||||
def has_member(self, user_id: UUID) -> bool:
|
||||
return user_id in self.member_ids
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Message:
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: tuple[UUID, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Attachment:
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
object_key: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
width: int
|
||||
height: int
|
||||
status: str
|
||||
error_code: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Keyset:
|
||||
created_at: datetime
|
||||
id: UUID
|
||||
|
||||
@property
|
||||
def order(self) -> tuple[datetime, int]:
|
||||
return (self.created_at, self.id.int)
|
||||
|
||||
@classmethod
|
||||
def from_item(cls, item) -> 'Keyset':
|
||||
return cls(created_at=item.created_at, id=item.id)
|
||||
|
||||
|
||||
def order_key(item) -> tuple[datetime, int]:
|
||||
return (item.created_at, item.id.int)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Page:
|
||||
items: tuple
|
||||
next: Optional[Keyset]
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""Сборка FastAPI приложения и подключение обязательных хранилищ."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy import text
|
||||
|
||||
from .api import errors
|
||||
from .api.deps import Services
|
||||
from .api.middleware import RequestContext
|
||||
from .api.routers import attachments, auth, chats, health, messages, users
|
||||
from .config import load_settings
|
||||
from .db import RedisSessionStore, create_engine, create_redis, create_session_factory
|
||||
from .repositories import AttachmentRepository, ChatRepository, MessageRepository, UserRepository
|
||||
from .services.attachments import AttachmentService
|
||||
from .services.chats import ChatService
|
||||
from .services.messages import MessageService
|
||||
from .services.users import UserService
|
||||
from .storage import S3Storage
|
||||
|
||||
logger = logging.getLogger('app')
|
||||
|
||||
DESCRIPTION = 'PostgreSQL хранит метаданные, Redis — сессии, приватный S3 — оригиналы вложений.'
|
||||
|
||||
|
||||
def configure_logging() -> None:
|
||||
logging.basicConfig(
|
||||
level=os.getenv('LOG_LEVEL', 'info').upper(),
|
||||
stream=sys.stdout,
|
||||
format='%(asctime)s %(levelname)s %(name)s %(message)s',
|
||||
force=True,
|
||||
)
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
configure_logging()
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(application: FastAPI):
|
||||
application.state.ready = False
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
redis = create_redis(settings.redis_url)
|
||||
storage = S3Storage(
|
||||
endpoint=settings.s3_endpoint,
|
||||
access_key=settings.s3_access_key_id,
|
||||
secret_key=settings.s3_secret_access_key,
|
||||
bucket=settings.s3_bucket,
|
||||
region=settings.s3_region,
|
||||
)
|
||||
try:
|
||||
async with engine.connect() as connection:
|
||||
await connection.execute(text('SELECT 1'))
|
||||
session_store = RedisSessionStore(redis, settings.session_ttl_seconds)
|
||||
await session_store.ping()
|
||||
await storage.ping()
|
||||
session_factory = create_session_factory(engine)
|
||||
user_repository = UserRepository(session_factory)
|
||||
chat_repository = ChatRepository(session_factory)
|
||||
message_repository = MessageRepository(session_factory)
|
||||
attachment_repository = AttachmentRepository(session_factory)
|
||||
application.state.services = Services(
|
||||
attachments=AttachmentService(
|
||||
attachment_repository,
|
||||
storage,
|
||||
settings.max_upload_bytes,
|
||||
settings.max_image_pixels,
|
||||
),
|
||||
users=UserService(user_repository),
|
||||
chats=ChatService(chat_repository),
|
||||
messages=MessageService(message_repository),
|
||||
sessions=session_store,
|
||||
)
|
||||
application.state.engine = engine
|
||||
application.state.redis = redis
|
||||
application.state.storage = storage
|
||||
application.state.ready = True
|
||||
logger.info('startup storage=postgres+s3 redis_sessions=true')
|
||||
yield
|
||||
finally:
|
||||
application.state.ready = False
|
||||
await redis.aclose()
|
||||
await engine.dispose()
|
||||
logger.info('shutdown complete')
|
||||
|
||||
api = FastAPI(
|
||||
title='MTUSI Messenger. LR 4',
|
||||
version='1.4.0',
|
||||
description=DESCRIPTION,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
api.router.redirect_slashes = False
|
||||
api.state.ready = False
|
||||
api.add_middleware(RequestContext)
|
||||
errors.install(api)
|
||||
for router in (
|
||||
health.router,
|
||||
users.router,
|
||||
auth.router,
|
||||
chats.router,
|
||||
messages.router,
|
||||
attachments.router,
|
||||
):
|
||||
api.include_router(router)
|
||||
return api
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,13 @@
|
||||
from .attachment import AttachmentRow, MessageAttachmentRow
|
||||
from .chat import ChatMemberRow, ChatRow
|
||||
from .message import MessageRow
|
||||
from .user import UserRow
|
||||
|
||||
__all__ = [
|
||||
'AttachmentRow',
|
||||
'ChatMemberRow',
|
||||
'ChatRow',
|
||||
'MessageAttachmentRow',
|
||||
'MessageRow',
|
||||
'UserRow',
|
||||
]
|
||||
@@ -0,0 +1,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class AttachmentRow(Base):
|
||||
__tablename__ = 'attachments'
|
||||
__table_args__ = (
|
||||
Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'),
|
||||
UniqueConstraint('object_key', name='attachments_object_key_key'),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
|
||||
owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
|
||||
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
object_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
width: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
height: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class MessageAttachmentRow(Base):
|
||||
__tablename__ = 'message_attachments'
|
||||
__table_args__ = (
|
||||
UniqueConstraint('message_id', 'position', name='message_attachments_message_position_key'),
|
||||
Index('message_attachments_attachment_idx', 'attachment_id'),
|
||||
)
|
||||
|
||||
message_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('messages.id', ondelete='CASCADE'), primary_key=True
|
||||
)
|
||||
attachment_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
@@ -0,0 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Index, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class ChatRow(Base):
|
||||
__tablename__ = 'chats'
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
title: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class ChatMemberRow(Base):
|
||||
__tablename__ = 'chat_members'
|
||||
__table_args__ = (Index('chat_members_user_chat_idx', 'user_id', 'chat_id'),)
|
||||
|
||||
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||
user_id: Mapped[UUID] = mapped_column(ForeignKey('users.id', ondelete='CASCADE'), primary_key=True)
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class MessageRow(Base):
|
||||
__tablename__ = 'messages'
|
||||
__table_args__ = (
|
||||
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
|
||||
UniqueConstraint(
|
||||
'sender_id',
|
||||
'chat_id',
|
||||
'idempotency_key',
|
||||
name='messages_sender_chat_idempotency_key',
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
|
||||
sender_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
|
||||
text: Mapped[str] = mapped_column(String(4000), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import DateTime, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
__tablename__ = 'users'
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
username: Mapped[str] = mapped_column(String(32), unique=True, nullable=False)
|
||||
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
@@ -0,0 +1,6 @@
|
||||
from .attachments import AttachmentRepository
|
||||
from .chats import ChatRepository
|
||||
from .messages import MessageRepository
|
||||
from .users import UserRepository
|
||||
|
||||
__all__ = ['AttachmentRepository', 'ChatRepository', 'MessageRepository', 'UserRepository']
|
||||
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.models import Attachment
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..models.chat import ChatMemberRow, ChatRow
|
||||
|
||||
|
||||
class AttachmentRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _to_domain(row: AttachmentRow) -> Attachment:
|
||||
return Attachment(
|
||||
id=row.id,
|
||||
chat_id=row.chat_id,
|
||||
owner_id=row.owner_id,
|
||||
filename=row.filename,
|
||||
object_key=row.object_key,
|
||||
content_type=row.content_type,
|
||||
size_bytes=row.size_bytes,
|
||||
sha256=row.sha256,
|
||||
width=row.width,
|
||||
height=row.height,
|
||||
status=row.status,
|
||||
error_code=row.error_code,
|
||||
created_at=row.created_at,
|
||||
updated_at=row.updated_at,
|
||||
)
|
||||
|
||||
async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool:
|
||||
async with self._sessions() as session:
|
||||
found = await session.scalar(
|
||||
select(ChatRow.id)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
|
||||
)
|
||||
return found is not None
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
object_key: str,
|
||||
content_type: str,
|
||||
size_bytes: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
now: datetime,
|
||||
) -> Attachment:
|
||||
row = AttachmentRow(
|
||||
id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=owner_id,
|
||||
filename=filename,
|
||||
object_key=object_key,
|
||||
content_type=content_type,
|
||||
size_bytes=size_bytes,
|
||||
sha256=sha256,
|
||||
width=width,
|
||||
height=height,
|
||||
status='ready',
|
||||
error_code=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
return self._to_domain(row)
|
||||
|
||||
async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None:
|
||||
async with self._sessions() as session:
|
||||
row = await session.scalar(
|
||||
select(AttachmentRow)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == AttachmentRow.chat_id)
|
||||
.where(
|
||||
AttachmentRow.id == attachment_id,
|
||||
ChatMemberRow.user_id == user_id,
|
||||
)
|
||||
)
|
||||
return self._to_domain(row) if row is not None else None
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Sequence
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, tuple_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.errors import Invalid
|
||||
from ..domain.models import Chat, Keyset
|
||||
from ..models.chat import ChatMemberRow, ChatRow
|
||||
from ..models.user import UserRow
|
||||
|
||||
|
||||
class ChatRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _to_domain(row: ChatRow, member_ids: Sequence[UUID]) -> Chat:
|
||||
return Chat(id=row.id, title=row.title, member_ids=tuple(member_ids), created_at=row.created_at)
|
||||
|
||||
@staticmethod
|
||||
async def _members(session: AsyncSession, chat_id: UUID) -> tuple[UUID, ...]:
|
||||
result = await session.scalars(
|
||||
select(ChatMemberRow.user_id)
|
||||
.where(ChatMemberRow.chat_id == chat_id)
|
||||
.order_by(ChatMemberRow.user_id)
|
||||
)
|
||||
return tuple(result)
|
||||
|
||||
async def create(self, chat_id: UUID, title: str, member_ids: Sequence[UUID], created_at: datetime) -> Chat:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
found = set(await session.scalars(select(UserRow.id).where(UserRow.id.in_(member_ids))))
|
||||
if len(found) != len(member_ids):
|
||||
raise Invalid(f'member_ids содержит неизвестных участников: {len(member_ids) - len(found)}')
|
||||
row = ChatRow(id=chat_id, title=title, created_at=created_at)
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
session.add_all(ChatMemberRow(chat_id=chat_id, user_id=user_id) for user_id in member_ids)
|
||||
await session.flush()
|
||||
return self._to_domain(row, member_ids)
|
||||
|
||||
async def get_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None:
|
||||
async with self._sessions() as session:
|
||||
row = await session.scalar(
|
||||
select(ChatRow)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
return self._to_domain(row, await self._members(session, row.id))
|
||||
|
||||
async def page_for_member(self, user_id: UUID, after: Keyset | None, limit: int) -> list[Chat]:
|
||||
async with self._sessions() as session:
|
||||
query = (
|
||||
select(ChatRow)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||
.where(ChatMemberRow.user_id == user_id)
|
||||
.order_by(ChatRow.created_at, ChatRow.id)
|
||||
.limit(limit)
|
||||
)
|
||||
if after is not None:
|
||||
query = query.where(tuple_(ChatRow.created_at, ChatRow.id) > tuple_(after.created_at, after.id))
|
||||
rows = list((await session.scalars(query)).all())
|
||||
return [self._to_domain(row, await self._members(session, row.id)) for row in rows]
|
||||
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Sequence
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, tuple_
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.errors import Conflict, NotFound
|
||||
from ..domain.models import Keyset, Message
|
||||
from ..models.attachment import AttachmentRow, MessageAttachmentRow
|
||||
from ..models.chat import ChatMemberRow, ChatRow
|
||||
from ..models.message import MessageRow
|
||||
|
||||
|
||||
class MessageRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _to_domain(row: MessageRow, attachment_ids: Sequence[UUID]) -> Message:
|
||||
return Message(
|
||||
id=row.id,
|
||||
chat_id=row.chat_id,
|
||||
sender_id=row.sender_id,
|
||||
text=row.text,
|
||||
created_at=row.created_at,
|
||||
attachment_ids=tuple(attachment_ids),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _attachment_ids(session: AsyncSession, message_id: UUID) -> tuple[UUID, ...]:
|
||||
result = await session.scalars(
|
||||
select(MessageAttachmentRow.attachment_id)
|
||||
.where(MessageAttachmentRow.message_id == message_id)
|
||||
.order_by(MessageAttachmentRow.position)
|
||||
)
|
||||
return tuple(result)
|
||||
|
||||
@staticmethod
|
||||
async def _accessible_chat_id(session: AsyncSession, chat_id: UUID, user_id: UUID) -> UUID | None:
|
||||
return await session.scalar(
|
||||
select(ChatRow.id)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
|
||||
)
|
||||
|
||||
async def create_for_member(
|
||||
self,
|
||||
message_id: UUID,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text: str,
|
||||
attachment_ids: Sequence[UUID],
|
||||
created_at: datetime,
|
||||
idempotency_key: str,
|
||||
) -> Message | None:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
accessible_chat_id = await self._accessible_chat_id(session, chat_id, sender_id)
|
||||
if accessible_chat_id is None:
|
||||
return None
|
||||
|
||||
if attachment_ids:
|
||||
found = {
|
||||
attachment_id: status
|
||||
for attachment_id, status in (
|
||||
await session.execute(
|
||||
select(AttachmentRow.id, AttachmentRow.status).where(
|
||||
AttachmentRow.id.in_(attachment_ids),
|
||||
AttachmentRow.chat_id == accessible_chat_id,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
}
|
||||
if set(found) != set(attachment_ids):
|
||||
raise NotFound('вложение не найдено')
|
||||
if any(status != 'ready' for status in found.values()):
|
||||
raise Conflict('оригинал вложения недоступен')
|
||||
|
||||
statement = (
|
||||
insert(MessageRow)
|
||||
.values(
|
||||
id=message_id,
|
||||
chat_id=accessible_chat_id,
|
||||
sender_id=sender_id,
|
||||
text=text,
|
||||
created_at=created_at,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
.on_conflict_do_nothing(
|
||||
index_elements=[
|
||||
MessageRow.sender_id,
|
||||
MessageRow.chat_id,
|
||||
MessageRow.idempotency_key,
|
||||
]
|
||||
)
|
||||
.returning(MessageRow.id)
|
||||
)
|
||||
created_id = await session.scalar(statement)
|
||||
if created_id is not None:
|
||||
row = await session.get(MessageRow, created_id)
|
||||
session.add_all(
|
||||
MessageAttachmentRow(
|
||||
message_id=created_id,
|
||||
attachment_id=attachment_id,
|
||||
position=position,
|
||||
)
|
||||
for position, attachment_id in enumerate(attachment_ids)
|
||||
)
|
||||
await session.flush()
|
||||
result_attachment_ids = tuple(attachment_ids)
|
||||
else:
|
||||
row = await session.scalar(
|
||||
select(MessageRow).where(
|
||||
MessageRow.sender_id == sender_id,
|
||||
MessageRow.chat_id == accessible_chat_id,
|
||||
MessageRow.idempotency_key == idempotency_key,
|
||||
)
|
||||
)
|
||||
if row is None:
|
||||
raise RuntimeError('результат идемпотентной операции не найден')
|
||||
result_attachment_ids = await self._attachment_ids(session, row.id)
|
||||
if row.text != text or result_attachment_ids != tuple(attachment_ids):
|
||||
raise Conflict('Idempotency-Key уже использован с другим сообщением')
|
||||
return self._to_domain(row, result_attachment_ids)
|
||||
|
||||
async def page_for_member(
|
||||
self, chat_id: UUID, user_id: UUID, after: Keyset | None, limit: int
|
||||
) -> list[Message] | None:
|
||||
async with self._sessions() as session:
|
||||
accessible_chat_id = await self._accessible_chat_id(session, chat_id, user_id)
|
||||
if accessible_chat_id is None:
|
||||
return None
|
||||
query = (
|
||||
select(MessageRow)
|
||||
.where(MessageRow.chat_id == accessible_chat_id)
|
||||
.order_by(MessageRow.created_at, MessageRow.id)
|
||||
.limit(limit)
|
||||
)
|
||||
if after is not None:
|
||||
query = query.where(tuple_(MessageRow.created_at, MessageRow.id) > tuple_(after.created_at, after.id))
|
||||
rows = list((await session.scalars(query)).all())
|
||||
return [
|
||||
self._to_domain(row, await self._attachment_ids(session, row.id))
|
||||
for row in rows
|
||||
]
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.errors import Conflict
|
||||
from ..domain.models import User
|
||||
from ..models.user import UserRow
|
||||
|
||||
|
||||
class UserRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _to_domain(row: UserRow) -> User:
|
||||
return User(id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at)
|
||||
|
||||
async def register(
|
||||
self, user_id: UUID, username: str, display_name: str, password_hash: str, created_at: datetime
|
||||
) -> User:
|
||||
row = UserRow(
|
||||
id=user_id,
|
||||
username=username,
|
||||
display_name=display_name,
|
||||
password_hash=password_hash,
|
||||
created_at=created_at,
|
||||
)
|
||||
try:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
except IntegrityError as exc:
|
||||
raise Conflict('username уже занят') from exc
|
||||
return self._to_domain(row)
|
||||
|
||||
async def get_by_id(self, user_id: UUID) -> User | None:
|
||||
async with self._sessions() as session:
|
||||
row = await session.get(UserRow, user_id)
|
||||
return self._to_domain(row) if row is not None else None
|
||||
|
||||
async def get_credentials(self, username: str) -> UserRow | None:
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(select(UserRow).where(UserRow.username == username))
|
||||
|
||||
@classmethod
|
||||
def to_domain(cls, row: UserRow) -> User:
|
||||
return cls._to_domain(row)
|
||||
@@ -0,0 +1,147 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from tempfile import SpooledTemporaryFile
|
||||
from typing import AsyncIterator, BinaryIO
|
||||
import unicodedata
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from ..domain.errors import (
|
||||
Conflict,
|
||||
Invalid,
|
||||
NotFound,
|
||||
PayloadTooLarge,
|
||||
Unavailable,
|
||||
UnsupportedMediaType,
|
||||
)
|
||||
from ..domain.models import Attachment, User
|
||||
from ..repositories import AttachmentRepository
|
||||
from ..storage import ObjectStream, S3Storage
|
||||
|
||||
logger = logging.getLogger('app.attachments')
|
||||
|
||||
MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
|
||||
|
||||
|
||||
def _validate_filename(filename: str) -> None:
|
||||
if not 1 <= len(filename) <= 255:
|
||||
raise Invalid('filename должен содержать от 1 до 255 символов')
|
||||
if any(
|
||||
character in {'/', '\\'} or unicodedata.category(character) == 'Cc'
|
||||
for character in filename
|
||||
):
|
||||
raise Invalid('filename содержит запрещённые символы')
|
||||
|
||||
|
||||
def _inspect_image(source: BinaryIO, declared_type: str, max_pixels: int) -> tuple[str, int, int]:
|
||||
try:
|
||||
source.seek(0)
|
||||
with Image.open(source) as image:
|
||||
actual_type = MIME_BY_FORMAT.get(image.format or '')
|
||||
if actual_type is None or actual_type != declared_type:
|
||||
raise UnsupportedMediaType('MIME не соответствует содержимому PNG/JPEG')
|
||||
width, height = image.size
|
||||
if width <= 0 or height <= 0:
|
||||
raise UnsupportedMediaType('изображение имеет неверные размеры')
|
||||
if width * height > max_pixels:
|
||||
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей')
|
||||
image.verify()
|
||||
except Image.DecompressionBombError as error:
|
||||
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') from error
|
||||
except (UnidentifiedImageError, OSError, SyntaxError) as error:
|
||||
raise UnsupportedMediaType('повреждённое или неподдерживаемое изображение') from error
|
||||
finally:
|
||||
source.seek(0)
|
||||
return actual_type, width, height
|
||||
|
||||
|
||||
class AttachmentService:
|
||||
def __init__(
|
||||
self,
|
||||
repository: AttachmentRepository,
|
||||
storage: S3Storage,
|
||||
max_upload_bytes: int,
|
||||
max_image_pixels: int,
|
||||
) -> None:
|
||||
self._repository = repository
|
||||
self._storage = storage
|
||||
self._max_upload_bytes = max_upload_bytes
|
||||
self._max_image_pixels = max_image_pixels
|
||||
|
||||
async def upload(
|
||||
self,
|
||||
actor: User,
|
||||
chat_id: UUID,
|
||||
filename: str,
|
||||
declared_type: str,
|
||||
chunks: AsyncIterator[bytes],
|
||||
) -> Attachment:
|
||||
_validate_filename(filename)
|
||||
if declared_type not in MIME_BY_FORMAT.values():
|
||||
raise UnsupportedMediaType('поддерживаются только image/png и image/jpeg')
|
||||
if not await self._repository.chat_exists_for_member(chat_id, actor.id):
|
||||
raise NotFound('чат не найден')
|
||||
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b') as source:
|
||||
async for chunk in chunks:
|
||||
size += len(chunk)
|
||||
if size > self._max_upload_bytes:
|
||||
raise PayloadTooLarge('файл превышает лимит 10 MiB')
|
||||
digest.update(chunk)
|
||||
source.write(chunk)
|
||||
if size == 0:
|
||||
raise UnsupportedMediaType('пустой файл не является изображением')
|
||||
|
||||
actual_type, width, height = await asyncio.to_thread(
|
||||
_inspect_image,
|
||||
source,
|
||||
declared_type,
|
||||
self._max_image_pixels,
|
||||
)
|
||||
attachment_id = uuid4()
|
||||
object_key = f'originals/{attachment_id}'
|
||||
await self._storage.put(object_key, source, size, actual_type)
|
||||
now = datetime.now(timezone.utc)
|
||||
try:
|
||||
return await self._repository.create(
|
||||
attachment_id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=actor.id,
|
||||
filename=filename,
|
||||
object_key=object_key,
|
||||
content_type=actual_type,
|
||||
size_bytes=size,
|
||||
sha256=digest.hexdigest(),
|
||||
width=width,
|
||||
height=height,
|
||||
now=now,
|
||||
)
|
||||
except Exception:
|
||||
try:
|
||||
await self._storage.remove(object_key)
|
||||
except Unavailable:
|
||||
logger.warning('failed to compensate orphan object key=%s', object_key)
|
||||
raise
|
||||
|
||||
async def get(self, actor: User, attachment_id: UUID) -> Attachment:
|
||||
attachment = await self._repository.get_for_member(attachment_id, actor.id)
|
||||
if attachment is None:
|
||||
raise NotFound('вложение не найдено')
|
||||
return attachment
|
||||
|
||||
async def content(self, actor: User, attachment_id: UUID) -> tuple[Attachment, ObjectStream]:
|
||||
attachment = await self.get(actor, attachment_id)
|
||||
if attachment.status != 'ready':
|
||||
raise Conflict('оригинал вложения недоступен')
|
||||
try:
|
||||
stream = await self._storage.get(attachment.object_key)
|
||||
except FileNotFoundError as error:
|
||||
raise Unavailable('оригинал вложения отсутствует в хранилище') from error
|
||||
return attachment, stream
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Sequence
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from ..domain.errors import Invalid, NotFound
|
||||
from ..domain.models import Chat, Keyset, Page, User
|
||||
from ..repositories import ChatRepository
|
||||
|
||||
|
||||
class ChatService:
|
||||
def __init__(self, repository: ChatRepository) -> None:
|
||||
self._repository = repository
|
||||
|
||||
async def create(self, actor: User, title: str, member_ids: Sequence[UUID]) -> Chat:
|
||||
if actor.id not in member_ids:
|
||||
raise Invalid('вызывающий должен входить в member_ids')
|
||||
return await self._repository.create(
|
||||
chat_id=uuid4(),
|
||||
title=title,
|
||||
member_ids=member_ids,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
async def get(self, actor: User, chat_id: UUID) -> Chat:
|
||||
chat = await self._repository.get_for_member(chat_id, actor.id)
|
||||
if chat is None:
|
||||
raise NotFound('чат не найден')
|
||||
return chat
|
||||
|
||||
async def page(self, actor: User, after: Optional[Keyset], limit: int) -> Page:
|
||||
rows = await self._repository.page_for_member(actor.id, after, limit + 1)
|
||||
items = tuple(rows[:limit])
|
||||
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Sequence
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from ..domain.errors import NotFound
|
||||
from ..domain.models import Keyset, Message, Page, User
|
||||
from ..repositories import MessageRepository
|
||||
|
||||
|
||||
class MessageService:
|
||||
def __init__(self, repository: MessageRepository) -> None:
|
||||
self._repository = repository
|
||||
|
||||
async def send(
|
||||
self,
|
||||
actor: User,
|
||||
chat_id: UUID,
|
||||
text: str,
|
||||
attachment_ids: Sequence[UUID],
|
||||
idempotency_key: str,
|
||||
) -> Message:
|
||||
message = await self._repository.create_for_member(
|
||||
message_id=uuid4(),
|
||||
chat_id=chat_id,
|
||||
sender_id=actor.id,
|
||||
text=text,
|
||||
attachment_ids=attachment_ids,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
if message is None:
|
||||
raise NotFound('чат не найден')
|
||||
return message
|
||||
|
||||
async def history(
|
||||
self, actor: User, chat_id: UUID, after: Optional[Keyset], limit: int
|
||||
) -> Page:
|
||||
rows = await self._repository.page_for_member(chat_id, actor.id, after, limit + 1)
|
||||
if rows is None:
|
||||
raise NotFound('чат не найден')
|
||||
items = tuple(rows[:limit])
|
||||
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from ..auth.passwords import hash_password, password_matches
|
||||
from ..domain.errors import Unauthorized
|
||||
from ..domain.models import User
|
||||
from ..repositories import UserRepository
|
||||
|
||||
|
||||
class UserService:
|
||||
def __init__(self, repository: UserRepository) -> None:
|
||||
self._repository = repository
|
||||
|
||||
async def register(self, username: str, display_name: str, password: str) -> User:
|
||||
return await self._repository.register(
|
||||
user_id=uuid4(),
|
||||
username=username,
|
||||
display_name=display_name,
|
||||
password_hash=hash_password(password),
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
async def identify(self, user_id: UUID) -> User:
|
||||
user = await self._repository.get_by_id(user_id)
|
||||
if user is None:
|
||||
raise Unauthorized('требуется действующая сессия')
|
||||
return user
|
||||
|
||||
async def authenticate(self, username: str, password: str) -> User | None:
|
||||
row = await self._repository.get_credentials(username)
|
||||
if row is None or not password_matches(row.password_hash, password):
|
||||
return None
|
||||
return self._repository.to_domain(row)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .s3 import ObjectStream, S3Storage, StoredObject
|
||||
|
||||
__all__ = ['ObjectStream', 'S3Storage', 'StoredObject']
|
||||
@@ -0,0 +1,132 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import certifi
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from minio import Minio
|
||||
from minio.error import InvalidResponseError, S3Error
|
||||
from urllib3 import PoolManager, Retry, Timeout
|
||||
from urllib3.exceptions import HTTPError
|
||||
|
||||
from ..domain.errors import Unavailable
|
||||
|
||||
|
||||
def _unavailable(error: Exception) -> Unavailable:
|
||||
return Unavailable('объектное хранилище недоступно')
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ObjectStream:
|
||||
response: object
|
||||
|
||||
def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]:
|
||||
try:
|
||||
while chunk := self.response.read(size):
|
||||
yield chunk
|
||||
finally:
|
||||
self.response.close()
|
||||
self.response.release_conn()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StoredObject:
|
||||
key: str
|
||||
last_modified: datetime
|
||||
|
||||
|
||||
class S3Storage:
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str,
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
bucket: str,
|
||||
region: str,
|
||||
) -> None:
|
||||
parsed = urlparse(endpoint)
|
||||
if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}:
|
||||
raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port')
|
||||
self._client = Minio(
|
||||
parsed.netloc,
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
secure=parsed.scheme == 'https',
|
||||
region=region,
|
||||
http_client=PoolManager(
|
||||
timeout=Timeout(connect=2, read=5),
|
||||
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
|
||||
cert_reqs='CERT_REQUIRED',
|
||||
ca_certs=certifi.where(),
|
||||
),
|
||||
)
|
||||
self.bucket = bucket
|
||||
|
||||
@staticmethod
|
||||
def _translate(error: Exception) -> None:
|
||||
if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}:
|
||||
raise FileNotFoundError from error
|
||||
if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)):
|
||||
raise _unavailable(error) from error
|
||||
raise error
|
||||
|
||||
async def ping(self) -> None:
|
||||
try:
|
||||
exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
if not exists:
|
||||
raise Unavailable('bucket объектного хранилища не создан')
|
||||
|
||||
async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None:
|
||||
source.seek(0)
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
self._client.put_object,
|
||||
self.bucket,
|
||||
key,
|
||||
source,
|
||||
length,
|
||||
content_type,
|
||||
)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
|
||||
async def get(self, key: str) -> ObjectStream:
|
||||
try:
|
||||
response = await asyncio.to_thread(self._client.get_object, self.bucket, key)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
return ObjectStream(response)
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
try:
|
||||
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
|
||||
return True
|
||||
except Exception as error:
|
||||
try:
|
||||
self._translate(error)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
raise
|
||||
|
||||
async def remove(self, key: str) -> None:
|
||||
try:
|
||||
await asyncio.to_thread(self._client.remove_object, self.bucket, key)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
|
||||
async def list(self, prefix: str = '') -> tuple[StoredObject, ...]:
|
||||
def collect() -> tuple[StoredObject, ...]:
|
||||
return tuple(
|
||||
StoredObject(item.object_name, item.last_modified)
|
||||
for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True)
|
||||
)
|
||||
|
||||
try:
|
||||
return await asyncio.to_thread(collect)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
@@ -0,0 +1 @@
|
||||
"""Операторские CLI-инструменты приложения."""
|
||||
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..storage import S3Storage
|
||||
|
||||
|
||||
async def cleanup(apply: bool, grace_seconds: int) -> int:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
try:
|
||||
sessions = create_session_factory(engine)
|
||||
async with sessions() as session:
|
||||
known = set(await session.scalars(select(AttachmentRow.object_key)))
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
removed = 0
|
||||
for item in await storage.list('originals/'):
|
||||
if item.key in known or item.last_modified > cutoff:
|
||||
continue
|
||||
action = 'DELETE' if apply else 'WOULD_DELETE'
|
||||
print(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
||||
if apply:
|
||||
await storage.remove(item.key)
|
||||
removed += 1
|
||||
print(
|
||||
f'summary mode={"apply" if apply else "dry-run"} '
|
||||
f'known={len(known)} removed={removed} grace_seconds={grace_seconds}'
|
||||
)
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description='Safely find old S3 objects without PostgreSQL metadata')
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--dry-run', action='store_true')
|
||||
mode.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--grace-seconds', type=int, default=3600)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 60:
|
||||
parser.error('--grace-seconds must be at least 60')
|
||||
raise SystemExit(asyncio.run(cleanup(args.apply, args.grace_seconds)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,250 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import tracemalloc
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Callable, Sequence
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..storage import S3Storage, StoredObject
|
||||
|
||||
MISSING_ERROR_CODE = 'original_missing'
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AttachmentSnapshot:
|
||||
id: UUID
|
||||
object_key: str
|
||||
status: str
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReconcilePlan:
|
||||
orphan: tuple[StoredObject, ...]
|
||||
fresh_orphan: tuple[StoredObject, ...]
|
||||
missing: tuple[AttachmentSnapshot, ...]
|
||||
fresh_missing: tuple[AttachmentSnapshot, ...]
|
||||
correct: int
|
||||
|
||||
|
||||
def build_plan(
|
||||
attachments: Sequence[AttachmentSnapshot],
|
||||
objects: Sequence[StoredObject],
|
||||
cutoff: datetime,
|
||||
) -> ReconcilePlan:
|
||||
"""Compare immutable PostgreSQL and S3 snapshots without changing either side."""
|
||||
known_keys = {attachment.object_key for attachment in attachments}
|
||||
object_keys = {item.key for item in objects}
|
||||
|
||||
unknown_objects = [item for item in objects if item.key not in known_keys]
|
||||
orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff)
|
||||
fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff)
|
||||
|
||||
ready_without_object = [
|
||||
attachment
|
||||
for attachment in attachments
|
||||
if attachment.status == 'ready' and attachment.object_key not in object_keys
|
||||
]
|
||||
missing = tuple(
|
||||
attachment for attachment in ready_without_object if attachment.updated_at <= cutoff
|
||||
)
|
||||
fresh_missing = tuple(
|
||||
attachment for attachment in ready_without_object if attachment.updated_at > cutoff
|
||||
)
|
||||
correct = sum(attachment.object_key in object_keys for attachment in attachments)
|
||||
return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct)
|
||||
|
||||
|
||||
async def _load_attachments(
|
||||
sessions: async_sessionmaker[AsyncSession],
|
||||
) -> tuple[AttachmentSnapshot, ...]:
|
||||
async with sessions() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
AttachmentRow.id,
|
||||
AttachmentRow.object_key,
|
||||
AttachmentRow.status,
|
||||
AttachmentRow.updated_at,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return tuple(AttachmentSnapshot(*row) for row in rows)
|
||||
|
||||
|
||||
async def _metadata_key_exists(
|
||||
sessions: async_sessionmaker[AsyncSession], object_key: str
|
||||
) -> bool:
|
||||
async with sessions() as session:
|
||||
attachment_id = await session.scalar(
|
||||
select(AttachmentRow.id).where(AttachmentRow.object_key == object_key).limit(1)
|
||||
)
|
||||
return attachment_id is not None
|
||||
|
||||
|
||||
async def reconcile(
|
||||
sessions: async_sessionmaker[AsyncSession],
|
||||
storage: S3Storage,
|
||||
*,
|
||||
apply: bool,
|
||||
grace_seconds: int,
|
||||
now: datetime | None = None,
|
||||
output: Callable[[str], None] = print,
|
||||
) -> dict[str, int | float | str]:
|
||||
started = time.perf_counter()
|
||||
tracemalloc.start()
|
||||
current_time = now or datetime.now(timezone.utc)
|
||||
cutoff = current_time - timedelta(seconds=grace_seconds)
|
||||
|
||||
attachments = await _load_attachments(sessions)
|
||||
objects = await storage.list('originals/')
|
||||
plan = build_plan(attachments, objects, cutoff)
|
||||
|
||||
for item in plan.fresh_orphan:
|
||||
output(
|
||||
f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}'
|
||||
)
|
||||
for attachment in plan.fresh_missing:
|
||||
output(
|
||||
'SKIP_FRESH_MISSING '
|
||||
f'attachment_id={attachment.id} key={attachment.object_key} '
|
||||
f'updated={attachment.updated_at.isoformat()}'
|
||||
)
|
||||
|
||||
removed = 0
|
||||
race_protected = 0
|
||||
for item in plan.orphan:
|
||||
action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN'
|
||||
if not apply:
|
||||
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
||||
continue
|
||||
# PostgreSQL and S3 cannot share a transaction. Re-read the authoritative
|
||||
# metadata immediately before a destructive action so a row created after
|
||||
# the initial snapshot protects its object.
|
||||
if await _metadata_key_exists(sessions, item.key):
|
||||
output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}')
|
||||
race_protected += 1
|
||||
continue
|
||||
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
||||
await storage.remove(item.key)
|
||||
removed += 1
|
||||
|
||||
marked_failed = 0
|
||||
for attachment in plan.missing:
|
||||
if not apply:
|
||||
output(
|
||||
f'WOULD_MARK_MISSING attachment_id={attachment.id} '
|
||||
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
|
||||
)
|
||||
continue
|
||||
|
||||
# The object may have been restored after list_objects(). Do not turn
|
||||
# healthy metadata into failed based only on the earlier snapshot.
|
||||
if await storage.exists(attachment.object_key):
|
||||
output(
|
||||
f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} '
|
||||
f'key={attachment.object_key}'
|
||||
)
|
||||
race_protected += 1
|
||||
continue
|
||||
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
update(AttachmentRow)
|
||||
.where(
|
||||
AttachmentRow.id == attachment.id,
|
||||
AttachmentRow.object_key == attachment.object_key,
|
||||
AttachmentRow.status == 'ready',
|
||||
AttachmentRow.updated_at == attachment.updated_at,
|
||||
)
|
||||
.values(
|
||||
status='failed',
|
||||
error_code=MISSING_ERROR_CODE,
|
||||
updated_at=current_time,
|
||||
)
|
||||
)
|
||||
changed = result.rowcount or 0
|
||||
if changed:
|
||||
output(
|
||||
f'MARK_MISSING attachment_id={attachment.id} '
|
||||
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
|
||||
)
|
||||
marked_failed += changed
|
||||
else:
|
||||
output(
|
||||
f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} '
|
||||
f'key={attachment.object_key}'
|
||||
)
|
||||
race_protected += 1
|
||||
|
||||
_, peak_bytes = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
summary: dict[str, int | float | str] = {
|
||||
'mode': 'apply' if apply else 'dry-run',
|
||||
'metadata': len(attachments),
|
||||
'objects': len(objects),
|
||||
'correct': plan.correct,
|
||||
'orphan': len(plan.orphan),
|
||||
'missing': len(plan.missing),
|
||||
'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing),
|
||||
'removed': removed,
|
||||
'marked_failed': marked_failed,
|
||||
'race_protected': race_protected,
|
||||
'grace_seconds': grace_seconds,
|
||||
'elapsed_ms': round((time.perf_counter() - started) * 1000, 3),
|
||||
'peak_memory_bytes': peak_bytes,
|
||||
}
|
||||
output(f'summary {json.dumps(summary, sort_keys=True)}')
|
||||
return summary
|
||||
|
||||
|
||||
async def run(apply: bool, grace_seconds: int) -> int:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
try:
|
||||
await reconcile(
|
||||
create_session_factory(engine),
|
||||
storage,
|
||||
apply=apply,
|
||||
grace_seconds=grace_seconds,
|
||||
)
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Reconcile attachment metadata in PostgreSQL with original objects in S3'
|
||||
)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--dry-run', action='store_true')
|
||||
mode.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--grace-seconds', type=int, default=3600)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 60:
|
||||
parser.error('--grace-seconds must be at least 60')
|
||||
raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,136 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from io import BytesIO
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
from sqlalchemy import delete, insert
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..models.chat import ChatMemberRow, ChatRow
|
||||
from ..models.user import UserRow
|
||||
from ..storage import S3Storage
|
||||
|
||||
USER_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:user')
|
||||
CHAT_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:chat')
|
||||
OBJECT_PREFIX = 'originals/reconciler-benchmark/'
|
||||
WORKERS = 16
|
||||
|
||||
|
||||
async def _put_objects(
|
||||
storage: S3Storage,
|
||||
keys: list[str],
|
||||
) -> None:
|
||||
for offset in range(0, len(keys), WORKERS):
|
||||
batch = keys[offset : offset + WORKERS]
|
||||
await asyncio.gather(
|
||||
*(storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream') for key in batch)
|
||||
)
|
||||
|
||||
|
||||
async def prepare(size: int) -> None:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
sessions = create_session_factory(engine)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
old = datetime.now(timezone.utc) - timedelta(hours=2)
|
||||
rows = [
|
||||
{
|
||||
'id': uuid5(NAMESPACE_URL, f'reconciler-benchmark:attachment:{index}'),
|
||||
'chat_id': CHAT_ID,
|
||||
'owner_id': USER_ID,
|
||||
'filename': f'benchmark-{index}.png',
|
||||
'object_key': f'{OBJECT_PREFIX}{index}',
|
||||
'content_type': 'image/png',
|
||||
'size_bytes': 1,
|
||||
'sha256': '0' * 64,
|
||||
'width': 1,
|
||||
'height': 1,
|
||||
'status': 'ready',
|
||||
'error_code': None,
|
||||
'created_at': old,
|
||||
'updated_at': old,
|
||||
}
|
||||
for index in range(size)
|
||||
]
|
||||
try:
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
session.add(UserRow(
|
||||
id=USER_ID,
|
||||
username=f'bench-{USER_ID.hex[:16]}',
|
||||
display_name='Reconciler benchmark',
|
||||
password_hash='not-used',
|
||||
created_at=old,
|
||||
))
|
||||
session.add(ChatRow(id=CHAT_ID, title='Reconciler benchmark', created_at=old))
|
||||
await session.flush()
|
||||
session.add(ChatMemberRow(chat_id=CHAT_ID, user_id=USER_ID))
|
||||
await session.flush()
|
||||
for offset in range(0, size, 500):
|
||||
await session.execute(insert(AttachmentRow), rows[offset : offset + 500])
|
||||
|
||||
await _put_objects(storage, [row['object_key'] for row in rows])
|
||||
print(f'prepared metadata={size} objects={size}')
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def cleanup() -> None:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
sessions = create_session_factory(engine)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
try:
|
||||
objects = await storage.list(OBJECT_PREFIX)
|
||||
keys = [item.key for item in objects]
|
||||
for offset in range(0, len(keys), WORKERS):
|
||||
await asyncio.gather(
|
||||
*(storage.remove(key) for key in keys[offset : offset + WORKERS])
|
||||
)
|
||||
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
await session.execute(delete(ChatRow).where(ChatRow.id == CHAT_ID))
|
||||
await session.execute(delete(UserRow).where(UserRow.id == USER_ID))
|
||||
print(f'cleaned objects={len(keys)}')
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Prepare and clean isolated PostgreSQL/S3 data for reconciler benchmarks'
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest='command', required=True)
|
||||
prepare_parser = subparsers.add_parser('prepare')
|
||||
prepare_parser.add_argument('--size', type=int, required=True)
|
||||
subparsers.add_parser('cleanup')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == 'prepare':
|
||||
if args.size < 1 or args.size > 10000:
|
||||
parser.error('--size must be between 1 and 10000')
|
||||
asyncio.run(prepare(args.size))
|
||||
else:
|
||||
asyncio.run(cleanup())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,120 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger_test
|
||||
POSTGRES_PASSWORD: messenger_test
|
||||
POSTGRES_DB: messenger_test
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger_test -d messenger_test"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
s3:
|
||||
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
|
||||
user: "0"
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: test-minio-admin
|
||||
MINIO_ROOT_PASSWORD: test-minio-root-password
|
||||
S3_BUCKET: messenger-attachments
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
s3-bootstrap:
|
||||
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
|
||||
mc mb --ignore-existing local/"$$S3_BUCKET"
|
||||
mc anonymous set none local/"$$S3_BUCKET"
|
||||
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
|
||||
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
|
||||
mc admin policy create local messenger-api /tmp/s3-api-policy.json
|
||||
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
|
||||
environment:
|
||||
MINIO_ROOT_USER: test-minio-admin
|
||||
MINIO_ROOT_PASSWORD: test-minio-root-password
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
volumes:
|
||||
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build: .
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
MAX_UPLOAD_BYTES: 10485760
|
||||
MAX_IMAGE_PIXELS: 40000000
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
tests:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.test
|
||||
environment:
|
||||
BASE_URL: http://api:8080
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.4-alpine
|
||||
cpus: "1.0"
|
||||
mem_limit: 512m
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
cpus: "0.25"
|
||||
mem_limit: 128m
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
s3:
|
||||
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
|
||||
user: "0"
|
||||
cpus: "0.5"
|
||||
mem_limit: 256m
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
s3-bootstrap:
|
||||
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
|
||||
cpus: "0.25"
|
||||
mem_limit: 128m
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
|
||||
mc mb --ignore-existing local/"$$S3_BUCKET"
|
||||
mc anonymous set none local/"$$S3_BUCKET"
|
||||
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
|
||||
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
|
||||
mc admin policy create local messenger-api /tmp/s3-api-policy.json
|
||||
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
volumes:
|
||||
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
cpus: "0.5"
|
||||
mem_limit: 256m
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build: .
|
||||
cpus: "0.5"
|
||||
mem_limit: 256m
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
expose:
|
||||
- "8080"
|
||||
deploy:
|
||||
replicas: 2
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
stop_grace_period: 10s
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.26-alpine
|
||||
cpus: "0.25"
|
||||
mem_limit: 128m
|
||||
ports:
|
||||
- "${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
stop_grace_period: 5s
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
@@ -0,0 +1,147 @@
|
||||
# Результаты проверки lab4
|
||||
|
||||
Дата последней перепроверки: 2026-11-05. Стенд: Apple M4 Pro, 12 CPU cores, 24 GB RAM, Docker Desktop.
|
||||
|
||||
## Версии runtime
|
||||
|
||||
```text
|
||||
Python 3.13.15
|
||||
PostgreSQL 17.4
|
||||
Redis 7.4.2
|
||||
HAProxy 3.2.26
|
||||
MinIO RELEASE.2026-10-23T20-06-51Z (commit 76c06ff1d2cc844050fdeb1d54bf8fa70db72278)
|
||||
MinIO Client DEVELOPMENT.GOGET
|
||||
```
|
||||
|
||||
MinIO и MinIO Client взяты из Chainguard Images и закреплены в Compose по multi-arch OCI index digest.
|
||||
|
||||
## Контракт и smoke
|
||||
|
||||
```text
|
||||
$ make check
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
$ make validate PYTHON=.venv/bin/python
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
|
||||
$ make test
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
```
|
||||
|
||||
## Негативные сценарии вложений
|
||||
|
||||
```text
|
||||
$ make test-attachments PYTHON=.venv/bin/python
|
||||
............ [100%]
|
||||
12 passed in 1.86s
|
||||
```
|
||||
|
||||
Проверены: превышение 10 MiB, повреждённое изображение, подмена MIME, превышение 40 млн пикселей до декодирования, опасные filename, доступ постороннего, attachment другого чата и семантика идемпотентности `attachment_ids`.
|
||||
|
||||
```text
|
||||
$ docker compose -f compose.tests.yaml run --rm --no-deps tests python -m pytest -q tests/test_attachment_compensation.py
|
||||
. [100%]
|
||||
1 passed
|
||||
```
|
||||
|
||||
Unit-тест внедряет ошибку сохранения метаданных после успешного S3 PUT и проверяет компенсационный DELETE того же object key.
|
||||
|
||||
Полный запуск в чистом изолированном Compose-проекте, включая миграции и bootstrap MinIO:
|
||||
|
||||
```text
|
||||
$ docker compose -p lab4-chainguard-tests -f compose.tests.yaml run --rm tests python -m pytest -q
|
||||
............. [100%]
|
||||
13 passed in 0.41s
|
||||
```
|
||||
|
||||
После добавления reconciler трека 2 итоговый запуск в отдельном Compose-проекте:
|
||||
|
||||
```text
|
||||
$ docker compose -p lab4-review3 -f compose.tests.yaml run --build --rm tests
|
||||
................. [100%]
|
||||
17 passed in 0.60s
|
||||
```
|
||||
|
||||
## Приватность и bootstrap
|
||||
|
||||
```text
|
||||
$ scripts/check-private-bucket.sh
|
||||
PASS: private bucket rejected anonymous request with HTTP 403
|
||||
```
|
||||
|
||||
MinIO root credentials получает только server/bootstrap. API использует отдельного пользователя с bucket-scoped policy. Host-порты S3 API и консоли не опубликованы. Bootstrap повторно завершался с code 0 при восстановлении API после failover и при restart стенда.
|
||||
|
||||
## Persistence и отказы
|
||||
|
||||
```text
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
```
|
||||
|
||||
Hook перезапустил API, PostgreSQL, Redis и MinIO с сохранением named volumes.
|
||||
|
||||
```text
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 1, "first_recovery_seconds": 2.333}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
```
|
||||
|
||||
```text
|
||||
$ make test-attachment-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
PASS: upload_instance=c18e79906d77 download_instance=662967386db9 recovery_seconds=3.912 failures=1 sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75
|
||||
```
|
||||
|
||||
В последнем сценарии остановлена именно реплика из `X-Instance-Id` ответа upload. Download выполнен через другую реплику и совпал побайтно.
|
||||
|
||||
```text
|
||||
$ scripts/check-s3-outage.sh
|
||||
PASS: API readiness returned structured 503 while S3 was unavailable
|
||||
```
|
||||
|
||||
Скрипт автоматически восстановил MinIO и дождался готовности стенда.
|
||||
|
||||
## Cleanup
|
||||
|
||||
```text
|
||||
$ python -m app.tools.cleanup_orphans --dry-run --grace-seconds 60
|
||||
summary mode=dry-run known=164 removed=0 grace_seconds=60
|
||||
```
|
||||
|
||||
Dry-run не менял данные. Применение требует явного `--apply`; значения grace period меньше 60 секунд запрещены.
|
||||
|
||||
## Reconciler PostgreSQL/S3 — трек 2
|
||||
|
||||
`tests/test_reconciler_integration.py` внедряет ошибки на обеих сторонах настоящего
|
||||
тестового стенда: старую metadata без объекта (`missing`) и объект без metadata
|
||||
(`orphan`). Контрольное вложение связано с сообщением.
|
||||
|
||||
```text
|
||||
$ docker compose -p lab4-review3 -f compose.tests.yaml run --rm tests \
|
||||
python -m pytest -q tests/test_reconciler.py tests/test_reconciler_integration.py
|
||||
... [100%]
|
||||
3 passed in 0.30s
|
||||
```
|
||||
|
||||
Проверено: dry-run ничего не меняет; apply удаляет один подтверждённый orphan и переводит
|
||||
один подтверждённый missing в `failed/original_missing`. Missing-объект, восстановленный
|
||||
после listing, missing-строка с изменившимся `updated_at` и orphan, для которого metadata
|
||||
появилась после снимка, получают `SKIP_*` и учитываются в `race_protected`. Отдельный
|
||||
свежий orphan переживает настоящий apply.
|
||||
Повторный apply выполняет 0 повторных исправлений. Объект используемого вложения и связь
|
||||
с сообщением сохранены.
|
||||
|
||||
Воспроизводимый end-to-end benchmark самостоятельно подготовил реальные данные в
|
||||
PostgreSQL и MinIO, а после измерения удалил их:
|
||||
|
||||
```text
|
||||
$ SIZES="100 1000 10000" RUNS=3 ./scripts/benchmark-reconciler.sh
|
||||
size=100: elapsed_ms=78.847,80.712,82.781; peak median=2620657
|
||||
size=1000: elapsed_ms=232.254,227.467,226.264; peak median=4205016
|
||||
size=10000: elapsed_ms=1701.005,1792.955,1733.545; peak median=8075853
|
||||
```
|
||||
|
||||
Медианы времени: 80,712; 227,467; 1733,545 мс. Скрипт готовит каждый размер один раз,
|
||||
делает три dry-run, затем удаляет benchmark-фикстуру. Измерение выполнено в отдельном
|
||||
Compose-проекте; фиксированные три пары, созданные интеграционными HTTP-проверками,
|
||||
присутствовали во всех точках одинаково и не влияют на вывод о масштабе.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"date": "2026-11-04",
|
||||
"profile": {
|
||||
"concurrency": 10,
|
||||
"uploads": 30,
|
||||
"file_size_bytes": 111817,
|
||||
"api_memory_limit_mib_each": 256
|
||||
},
|
||||
"result": {
|
||||
"successful_uploads": 30,
|
||||
"observed_api_instances": 2,
|
||||
"duration_seconds": 0.382,
|
||||
"max_latency_seconds": 0.3,
|
||||
"peak_memory_mib": {
|
||||
"api_replica_1": 68.5,
|
||||
"api_replica_2": 68.33
|
||||
}
|
||||
},
|
||||
"limitations": "Closed-loop profile with a small PNG; docker stats is sampled and may miss a shorter transient peak."
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout check 2s
|
||||
|
||||
resolvers docker
|
||||
nameserver docker_dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api_backend
|
||||
|
||||
backend api_backend
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||
server-template api 2 api:8080
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::BUCKET_NAME"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::BUCKET_NAME/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,2 +1,3 @@
|
||||
# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
|
||||
openapi-spec-validator==0.7.2
|
||||
pytest==8.3.5
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn[standard]==0.34.2
|
||||
SQLAlchemy[asyncio]==2.0.40
|
||||
asyncpg==0.30.0
|
||||
alembic==1.15.2
|
||||
redis==5.2.1
|
||||
argon2-cffi==25.1.0
|
||||
minio==7.2.20
|
||||
Pillow==13.0.0
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
sizes="${SIZES:-${SIZE:-100 1000 10000}}"
|
||||
runs="${RUNS:-3}"
|
||||
|
||||
case "$runs" in
|
||||
''|*[!0-9]*) printf '%s\n' 'RUNS must be a positive integer' >&2; exit 2 ;;
|
||||
0) printf '%s\n' 'RUNS must be a positive integer' >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
cleanup() {
|
||||
docker compose exec -T api python -m app.tools.seed_reconciler_benchmark cleanup
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
for size in $sizes; do
|
||||
cleanup
|
||||
docker compose exec -T api python -m app.tools.seed_reconciler_benchmark prepare --size "$size"
|
||||
run=1
|
||||
while [ "$run" -le "$runs" ]; do
|
||||
printf 'benchmark size=%s run=%s/%s\n' "$size" "$run" "$runs"
|
||||
docker compose exec -T api python -m app.tools.reconcile_attachments --dry-run
|
||||
run=$((run + 1))
|
||||
done
|
||||
done
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
project_dir=$(dirname -- "$script_dir")
|
||||
cd "$project_dir"
|
||||
|
||||
status=$(docker compose --env-file .env exec -T api python -c '
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
url = "http://s3:9000/" + os.environ["S3_BUCKET"] + "/"
|
||||
try:
|
||||
response = urllib.request.urlopen(url, timeout=5)
|
||||
except urllib.error.HTTPError as error:
|
||||
print(error.code)
|
||||
else:
|
||||
print(response.status)
|
||||
')
|
||||
|
||||
case "$status" in
|
||||
401|403) printf '%s\n' "PASS: private bucket rejected anonymous request with HTTP $status" ;;
|
||||
*) printf '%s\n' "FAIL: anonymous bucket request returned HTTP $status" >&2; exit 1 ;;
|
||||
esac
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
project_dir=$(dirname -- "$script_dir")
|
||||
cd "$project_dir"
|
||||
|
||||
restore() {
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 s3 api lb >/dev/null
|
||||
}
|
||||
trap restore EXIT HUP INT TERM
|
||||
|
||||
docker compose --env-file .env stop s3 >/dev/null
|
||||
docker compose --env-file .env exec -T api python -c '
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=25)
|
||||
except urllib.error.HTTPError as error:
|
||||
body = json.loads(error.read())
|
||||
assert error.code == 503, error.code
|
||||
assert body["error"]["code"] == "unavailable", body
|
||||
else:
|
||||
raise SystemExit("readiness unexpectedly succeeded while S3 was stopped")
|
||||
'
|
||||
|
||||
printf '%s\n' 'PASS: API readiness returned structured 503 while S3 was unavailable'
|
||||
Executable
+109
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run concurrent uploads while sampling API container memory with docker stats."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
UNITS = {
|
||||
'B': 1,
|
||||
'kB': 1000,
|
||||
'KB': 1000,
|
||||
'KiB': 1024,
|
||||
'MB': 1000**2,
|
||||
'MiB': 1024**2,
|
||||
'GB': 1000**3,
|
||||
'GiB': 1024**3,
|
||||
}
|
||||
|
||||
|
||||
def bytes_value(value: str) -> int:
|
||||
match = re.fullmatch(r'([0-9.]+)([A-Za-z]+)', value.strip())
|
||||
if match is None or match.group(2) not in UNITS:
|
||||
raise ValueError(f'unsupported docker memory value: {value}')
|
||||
return round(float(match.group(1)) * UNITS[match.group(2)])
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--uploads', type=int, default=30)
|
||||
args = parser.parse_args()
|
||||
|
||||
container_ids = subprocess.check_output(
|
||||
['docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api'],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
).split()
|
||||
if len(container_ids) < 2:
|
||||
raise SystemExit('expected at least two API containers')
|
||||
|
||||
peaks: dict[str, int] = {}
|
||||
stop = threading.Event()
|
||||
ready = threading.Event()
|
||||
|
||||
def collect() -> None:
|
||||
while not stop.is_set():
|
||||
output = subprocess.check_output(
|
||||
['docker', 'stats', '--no-stream', '--format', '{{json .}}', *container_ids],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
)
|
||||
for line in output.splitlines():
|
||||
sample = json.loads(line)
|
||||
name, usage = sample['Name'], sample['MemUsage']
|
||||
current = bytes_value(usage.split('/', 1)[0])
|
||||
peaks[name] = max(peaks.get(name, 0), current)
|
||||
ready.set()
|
||||
|
||||
reader = threading.Thread(target=collect, daemon=True)
|
||||
reader.start()
|
||||
if not ready.wait(timeout=15):
|
||||
raise SystemExit('docker stats did not produce an initial sample')
|
||||
try:
|
||||
load = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
'tests/upload_load.py',
|
||||
'--base-url',
|
||||
args.base_url,
|
||||
'--concurrency',
|
||||
str(args.concurrency),
|
||||
'--uploads',
|
||||
str(args.uploads),
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
finally:
|
||||
stop.set()
|
||||
reader.join(timeout=15)
|
||||
|
||||
if not peaks:
|
||||
raise SystemExit('docker stats produced no samples; increase --uploads')
|
||||
print(load.stdout.strip())
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
'peak_memory_bytes': peaks,
|
||||
'peak_memory_mib': {
|
||||
name: round(value / 1024 / 1024, 2) for name, value in peaks.items()
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
project_dir=$(dirname -- "$script_dir")
|
||||
cd "$project_dir"
|
||||
|
||||
# Останавливаем публичный маршрут до перезапуска общих хранилищ.
|
||||
docker compose --env-file .env stop lb api >/dev/null
|
||||
docker compose --env-file .env restart postgres redis s3 >/dev/null
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 >/dev/null
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
project_dir=$(dirname -- "$script_dir")
|
||||
cd "$project_dir"
|
||||
|
||||
target_container=''
|
||||
for container_id in $(docker compose --env-file .env ps -q api); do
|
||||
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
|
||||
target_container=$container_id
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$target_container" ]; then
|
||||
printf '%s\n' 'API instance from TARGET_INSTANCE_ID was not found' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker stop --time 5 "$target_container" >/dev/null
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stop the API replica that accepted upload and download through another replica."""
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from client import Client, ROOT, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--action-script', required=True)
|
||||
|
||||
args = arguments('Attachment API-replica failover', configure)
|
||||
script = Path(args.action_script).expanduser().resolve()
|
||||
check(script.is_file() and os.access(script, os.X_OK), 'action script must be executable')
|
||||
client = Client(args.base_url, args.ca_file, timeout=3)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
uploaded = client.call(
|
||||
'POST',
|
||||
f'/api/v1/chats/{chat["id"]}/attachments?filename=failover.png',
|
||||
201,
|
||||
actor=alice,
|
||||
raw=content,
|
||||
headers={'Content-Type': 'image/png'},
|
||||
schema='Attachment',
|
||||
)
|
||||
target = uploaded['headers']['X-Instance-Id']
|
||||
env = dict(os.environ)
|
||||
env['TARGET_INSTANCE_ID'] = target
|
||||
started = time.monotonic()
|
||||
subprocess.run([str(script)], env=env, check=True)
|
||||
|
||||
deadline = time.monotonic() + 15
|
||||
failures = 0
|
||||
while True:
|
||||
try:
|
||||
downloaded = client.call(
|
||||
'GET',
|
||||
f'/api/v1/attachments/{uploaded["json"]["id"]}/content',
|
||||
actor=bob,
|
||||
)
|
||||
if downloaded['headers']['X-Instance-Id'] != target:
|
||||
break
|
||||
except Exception:
|
||||
failures += 1
|
||||
check(time.monotonic() < deadline, 'download did not recover through another replica')
|
||||
time.sleep(0.2)
|
||||
|
||||
check(downloaded['bytes'] == content, 'downloaded file differs after replica failure')
|
||||
print(
|
||||
'PASS: upload_instance=%s download_instance=%s recovery_seconds=%.3f failures=%d sha256=%s'
|
||||
% (
|
||||
target,
|
||||
downloaded['headers']['X-Instance-Id'],
|
||||
time.monotonic() - started,
|
||||
failures,
|
||||
hashlib.sha256(content).hexdigest(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Проверка компенсации объекта при ошибке commit метаданных."""
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.domain.errors import Conflict
|
||||
from app.domain.models import Attachment
|
||||
from app.domain.models import User
|
||||
from app.services.attachments import AttachmentService
|
||||
|
||||
|
||||
class FailingRepository:
|
||||
async def chat_exists_for_member(self, chat_id, user_id):
|
||||
return True
|
||||
|
||||
async def create(self, **kwargs):
|
||||
raise RuntimeError('injected database commit failure')
|
||||
|
||||
|
||||
class RecordingStorage:
|
||||
def __init__(self):
|
||||
self.put_key = None
|
||||
self.removed_key = None
|
||||
|
||||
async def put(self, key, source, length, content_type):
|
||||
self.put_key = key
|
||||
|
||||
async def remove(self, key):
|
||||
self.removed_key = key
|
||||
|
||||
|
||||
class FailedAttachmentRepository:
|
||||
def __init__(self, attachment):
|
||||
self.attachment = attachment
|
||||
|
||||
async def get_for_member(self, attachment_id, user_id):
|
||||
return self.attachment
|
||||
|
||||
|
||||
class StorageThatMustNotBeRead:
|
||||
async def get(self, key):
|
||||
raise AssertionError('failed attachment must not read S3')
|
||||
|
||||
|
||||
def test_uploaded_object_is_removed_when_metadata_commit_fails():
|
||||
"""Удаляет объект после ошибки сохранения метаданных."""
|
||||
storage = RecordingStorage()
|
||||
service = AttachmentService(FailingRepository(), storage, 10 * 1024 * 1024, 40_000_000)
|
||||
actor = User(uuid4(), 'tester', 'Tester', datetime.now(timezone.utc))
|
||||
content = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
|
||||
|
||||
async def chunks():
|
||||
yield content
|
||||
|
||||
with pytest.raises(RuntimeError, match='injected database commit failure'):
|
||||
asyncio.run(
|
||||
service.upload(
|
||||
actor,
|
||||
uuid4(),
|
||||
'sample.png',
|
||||
'image/png',
|
||||
chunks(),
|
||||
)
|
||||
)
|
||||
|
||||
assert storage.put_key is not None
|
||||
assert storage.removed_key == storage.put_key
|
||||
|
||||
|
||||
def test_failed_attachment_content_returns_conflict_without_s3_read():
|
||||
"""Возвращает конфликт для ошибочного вложения без чтения S3."""
|
||||
now = datetime.now(timezone.utc)
|
||||
attachment = Attachment(
|
||||
id=uuid4(),
|
||||
chat_id=uuid4(),
|
||||
owner_id=uuid4(),
|
||||
filename='missing.png',
|
||||
object_key=f'originals/{uuid4()}',
|
||||
content_type='image/png',
|
||||
size_bytes=1,
|
||||
sha256='0' * 64,
|
||||
width=1,
|
||||
height=1,
|
||||
status='failed',
|
||||
error_code='original_missing',
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
service = AttachmentService(
|
||||
FailedAttachmentRepository(attachment), StorageThatMustNotBeRead(), 1, 1
|
||||
)
|
||||
actor = User(uuid4(), 'tester', 'Tester', now)
|
||||
|
||||
with pytest.raises(Conflict, match='недоступен'):
|
||||
asyncio.run(service.content(actor, attachment.id))
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Негативные интеграционные проверки вложений для уровня 4."""
|
||||
import struct
|
||||
from urllib.parse import quote
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.client import Client, ROOT
|
||||
|
||||
|
||||
@pytest.fixture(scope='module')
|
||||
def scenario():
|
||||
client = Client()
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
return client, alice, bob, eve, chat
|
||||
|
||||
|
||||
def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=201):
|
||||
return client.call(
|
||||
'POST',
|
||||
f'/api/v1/chats/{chat["id"]}/attachments?filename={quote(filename, safe="")}',
|
||||
expected,
|
||||
actor=actor,
|
||||
raw=content,
|
||||
headers={'Content-Type': content_type},
|
||||
schema='Attachment' if expected == 201 else None,
|
||||
)
|
||||
|
||||
|
||||
def oversized_dimensions_png(content: bytes) -> bytes:
|
||||
data = bytearray(content)
|
||||
data[16:20] = struct.pack('>I', 10_000)
|
||||
data[20:24] = struct.pack('>I', 5_000)
|
||||
data[29:33] = struct.pack('>I', zlib.crc32(data[12:29]) & 0xFFFFFFFF)
|
||||
return bytes(data)
|
||||
|
||||
|
||||
def test_file_larger_than_10_mib_is_rejected(scenario):
|
||||
"""Отклоняет файл размером больше 10 MiB."""
|
||||
client, alice, _bob, _eve, chat = scenario
|
||||
upload(client, alice, chat, b'x' * (10 * 1024 * 1024 + 1), expected=413)
|
||||
|
||||
|
||||
def test_corrupted_image_is_rejected(scenario):
|
||||
"""Отклоняет повреждённое изображение."""
|
||||
client, alice, _bob, _eve, chat = scenario
|
||||
upload(client, alice, chat, b'not-a-png', expected=415)
|
||||
|
||||
|
||||
def test_mime_mismatch_is_rejected(scenario):
|
||||
"""Отклоняет MIME, не совпадающий с содержимым."""
|
||||
client, alice, _bob, _eve, chat = scenario
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
upload(client, alice, chat, content, content_type='image/jpeg', expected=415)
|
||||
|
||||
|
||||
def test_pixel_limit_is_checked_before_decode(scenario):
|
||||
"""Проверяет лимит пикселей до полной декомпрессии."""
|
||||
client, alice, _bob, _eve, chat = scenario
|
||||
content = oversized_dimensions_png((ROOT / 'tests/fixtures/sample.png').read_bytes())
|
||||
upload(client, alice, chat, content, expected=413)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'filename',
|
||||
['../escape.png', 'folder\\escape.png', 'bad\x00name.png', 'bad\x85name.png'],
|
||||
)
|
||||
def test_dangerous_filename_is_rejected(scenario, filename):
|
||||
"""Отклоняет опасное отображаемое имя файла."""
|
||||
client, alice, _bob, _eve, chat = scenario
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
upload(client, alice, chat, content, filename=filename, expected=400)
|
||||
|
||||
|
||||
def test_outsider_cannot_read_attachment(scenario):
|
||||
"""Скрывает метаданные и содержимое от постороннего пользователя."""
|
||||
client, alice, _bob, eve, chat = scenario
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
attachment = upload(client, alice, chat, content)['json']
|
||||
client.call('GET', f'/api/v1/attachments/{attachment["id"]}', 404, actor=eve)
|
||||
client.call('GET', f'/api/v1/attachments/{attachment["id"]}/content', 404, actor=eve)
|
||||
|
||||
|
||||
def test_attachment_from_another_chat_cannot_be_linked(scenario):
|
||||
"""Запрещает прикреплять вложение другого чата."""
|
||||
client, alice, bob, _eve, chat = scenario
|
||||
other_chat = client.call(
|
||||
'POST',
|
||||
'/api/v1/chats',
|
||||
201,
|
||||
actor=alice,
|
||||
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]},
|
||||
schema='Chat',
|
||||
)['json']
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
attachment = upload(client, alice, other_chat, content)['json']
|
||||
client.call(
|
||||
'POST',
|
||||
f'/api/v1/chats/{chat["id"]}/messages',
|
||||
404,
|
||||
actor=alice,
|
||||
body={'text': 'чужое вложение', 'attachment_ids': [attachment['id']]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
def test_idempotency_distinguishes_attachment_list(scenario):
|
||||
"""Включает список вложений в сравнение идемпотентного запроса."""
|
||||
client, alice, _bob, _eve, chat = scenario
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
attachment = upload(client, alice, chat, content)['json']
|
||||
key = uuid.uuid4().hex
|
||||
first = client.message(
|
||||
alice,
|
||||
chat,
|
||||
'идемпотентное вложение',
|
||||
key=key,
|
||||
attachments=[attachment['id']],
|
||||
)['json']
|
||||
replay = client.message(
|
||||
alice,
|
||||
chat,
|
||||
'идемпотентное вложение',
|
||||
key=key,
|
||||
attachments=[attachment['id']],
|
||||
)['json']
|
||||
assert replay == first
|
||||
client.call(
|
||||
'POST',
|
||||
f'/api/v1/chats/{chat["id"]}/messages',
|
||||
409,
|
||||
actor=alice,
|
||||
body={'text': 'идемпотентное вложение', 'attachment_ids': []},
|
||||
headers={'Idempotency-Key': key},
|
||||
)
|
||||
|
||||
|
||||
def test_missing_attachment_ids_equals_empty_list_for_idempotency(scenario):
|
||||
"""Считает отсутствие attachment_ids эквивалентным пустому списку."""
|
||||
client, alice, _bob, _eve, chat = scenario
|
||||
key = uuid.uuid4().hex
|
||||
first = client.message(alice, chat, 'обычный текст', key=key)['json']
|
||||
replay = client.call(
|
||||
'POST',
|
||||
f'/api/v1/chats/{chat["id"]}/messages',
|
||||
201,
|
||||
actor=alice,
|
||||
body={'text': 'обычный текст', 'attachment_ids': []},
|
||||
headers={'Idempotency-Key': key},
|
||||
schema='Message',
|
||||
)['json']
|
||||
assert replay == first
|
||||
@@ -0,0 +1,52 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
from app.storage import StoredObject
|
||||
from app.tools.reconcile_attachments import AttachmentSnapshot, build_plan
|
||||
|
||||
|
||||
def test_plan_classifies_both_sides_and_protects_fresh_items():
|
||||
"""Классифицирует orphan/missing и защищает свежие расхождения."""
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(minutes=1)
|
||||
correct_key = f'originals/{uuid4()}'
|
||||
missing = AttachmentSnapshot(uuid4(), f'originals/{uuid4()}', 'ready', cutoff)
|
||||
fresh_missing = AttachmentSnapshot(uuid4(), f'originals/{uuid4()}', 'ready', now)
|
||||
already_failed = AttachmentSnapshot(uuid4(), f'originals/{uuid4()}', 'failed', cutoff)
|
||||
attachments = (
|
||||
AttachmentSnapshot(uuid4(), correct_key, 'ready', cutoff),
|
||||
missing,
|
||||
fresh_missing,
|
||||
already_failed,
|
||||
)
|
||||
old_orphan = StoredObject(f'originals/{uuid4()}', cutoff)
|
||||
fresh_orphan = StoredObject(f'originals/{uuid4()}', now)
|
||||
|
||||
plan = build_plan(
|
||||
attachments,
|
||||
(StoredObject(correct_key, cutoff), old_orphan, fresh_orphan),
|
||||
cutoff,
|
||||
)
|
||||
|
||||
assert plan.correct == 1
|
||||
assert plan.orphan == (old_orphan,)
|
||||
assert plan.fresh_orphan == (fresh_orphan,)
|
||||
assert plan.missing == (missing,)
|
||||
assert plan.fresh_missing == (fresh_missing,)
|
||||
|
||||
|
||||
def test_failed_metadata_still_protects_its_object_from_orphan_cleanup():
|
||||
"""Проверяет защиту объекта существующей failed-metadata."""
|
||||
now = datetime.now(timezone.utc)
|
||||
key = f'originals/{uuid4()}'
|
||||
failed = AttachmentSnapshot(uuid4(), key, 'failed', now - timedelta(hours=2))
|
||||
|
||||
plan = build_plan(
|
||||
(failed,),
|
||||
(StoredObject(key, now - timedelta(hours=2)),),
|
||||
now - timedelta(hours=1),
|
||||
)
|
||||
|
||||
assert plan.correct == 1
|
||||
assert plan.orphan == ()
|
||||
assert plan.missing == ()
|
||||
@@ -0,0 +1,273 @@
|
||||
"""Real PostgreSQL + S3 fault injection for grade 5, track 2."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from io import BytesIO
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app.config import load_settings
|
||||
from app.db import create_engine, create_session_factory
|
||||
from app.models.attachment import AttachmentRow, MessageAttachmentRow
|
||||
from app.models.chat import ChatMemberRow, ChatRow
|
||||
from app.models.message import MessageRow
|
||||
from app.models.user import UserRow
|
||||
from app.storage import S3Storage
|
||||
from app.tools.reconcile_attachments import MISSING_ERROR_CODE, reconcile
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.getenv('DATABASE_URL'), reason='requires the compose PostgreSQL and S3 services'
|
||||
)
|
||||
|
||||
|
||||
class AfterListStorage:
|
||||
def __init__(self, storage, after_list):
|
||||
self._storage = storage
|
||||
self._after_list = after_list
|
||||
|
||||
async def list(self, prefix=''):
|
||||
snapshot = await self._storage.list(prefix)
|
||||
if self._after_list is not None:
|
||||
after_list, self._after_list = self._after_list, None
|
||||
await after_list()
|
||||
return snapshot
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._storage, name)
|
||||
|
||||
|
||||
def test_dry_run_apply_and_repeat_are_safe():
|
||||
"""Проверяет пробный, изменяющий и повторный запуски."""
|
||||
asyncio.run(_check_dry_run_apply_and_repeat_are_safe())
|
||||
|
||||
|
||||
async def _check_dry_run_apply_and_repeat_are_safe():
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
sessions = create_session_factory(engine)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
now = datetime.now(timezone.utc)
|
||||
old = now - timedelta(hours=2)
|
||||
simulated_after_grace = now + timedelta(seconds=61)
|
||||
user_id, chat_id, message_id = uuid4(), uuid4(), uuid4()
|
||||
correct_id, missing_id, restored_id = uuid4(), uuid4(), uuid4()
|
||||
changed_id, appeared_id = uuid4(), uuid4()
|
||||
correct_key = f'originals/reconcile-correct-{uuid4()}'
|
||||
orphan_key = f'originals/reconcile-orphan-{uuid4()}'
|
||||
restored_key = f'originals/reconcile-restored-{uuid4()}'
|
||||
changed_key = f'originals/reconcile-changed-{uuid4()}'
|
||||
appeared_key = f'originals/reconcile-appeared-{uuid4()}'
|
||||
fresh_key = f'originals/reconcile-fresh-{uuid4()}'
|
||||
|
||||
try:
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
session.add(UserRow(
|
||||
id=user_id,
|
||||
username=f'reconcile-{user_id.hex[:16]}',
|
||||
display_name='Reconciler test',
|
||||
password_hash='not-used',
|
||||
created_at=old,
|
||||
))
|
||||
session.add(ChatRow(id=chat_id, title='Reconciler test', created_at=old))
|
||||
await session.flush()
|
||||
session.add(ChatMemberRow(chat_id=chat_id, user_id=user_id))
|
||||
for attachment_id, key in (
|
||||
(correct_id, correct_key),
|
||||
(missing_id, f'originals/reconcile-missing-{uuid4()}'),
|
||||
(restored_id, restored_key),
|
||||
(changed_id, changed_key),
|
||||
):
|
||||
session.add(AttachmentRow(
|
||||
id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=user_id,
|
||||
filename='test.png',
|
||||
object_key=key,
|
||||
content_type='image/png',
|
||||
size_bytes=1,
|
||||
sha256='0' * 64,
|
||||
width=1,
|
||||
height=1,
|
||||
status='ready',
|
||||
error_code=None,
|
||||
created_at=old,
|
||||
updated_at=old,
|
||||
))
|
||||
await session.flush()
|
||||
session.add(MessageRow(
|
||||
id=message_id,
|
||||
chat_id=chat_id,
|
||||
sender_id=user_id,
|
||||
text='used attachment must survive',
|
||||
created_at=old,
|
||||
idempotency_key=f'reconcile-{message_id}',
|
||||
))
|
||||
await session.flush()
|
||||
session.add(MessageAttachmentRow(
|
||||
message_id=message_id,
|
||||
attachment_id=correct_id,
|
||||
position=0,
|
||||
))
|
||||
|
||||
for key in (correct_key, orphan_key):
|
||||
await storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream')
|
||||
|
||||
dry_run = await reconcile(
|
||||
sessions,
|
||||
storage,
|
||||
apply=False,
|
||||
grace_seconds=60,
|
||||
now=simulated_after_grace,
|
||||
output=lambda _line: None,
|
||||
)
|
||||
assert dry_run['orphan'] == 1
|
||||
assert dry_run['missing'] == 3
|
||||
assert dry_run['removed'] == 0
|
||||
assert dry_run['marked_failed'] == 0
|
||||
assert {item.key for item in await storage.list('originals/')} >= {
|
||||
correct_key,
|
||||
orphan_key,
|
||||
}
|
||||
|
||||
async def restore_missing_after_snapshot():
|
||||
await storage.put(restored_key, BytesIO(b'x'), 1, 'application/octet-stream')
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
await session.execute(
|
||||
update(AttachmentRow)
|
||||
.where(AttachmentRow.id == changed_id)
|
||||
.values(updated_at=now)
|
||||
)
|
||||
|
||||
applied = await reconcile(
|
||||
sessions,
|
||||
AfterListStorage(storage, restore_missing_after_snapshot),
|
||||
apply=True,
|
||||
grace_seconds=60,
|
||||
now=simulated_after_grace,
|
||||
output=lambda _line: None,
|
||||
)
|
||||
assert applied['removed'] == 1
|
||||
assert applied['marked_failed'] == 1
|
||||
assert applied['race_protected'] == 2
|
||||
|
||||
# Make the concurrently changed metadata healthy before checking that
|
||||
# the next complete run has no work left.
|
||||
await storage.put(changed_key, BytesIO(b'x'), 1, 'application/octet-stream')
|
||||
|
||||
repeated = await reconcile(
|
||||
sessions,
|
||||
storage,
|
||||
apply=True,
|
||||
grace_seconds=60,
|
||||
now=simulated_after_grace,
|
||||
output=lambda _line: None,
|
||||
)
|
||||
assert repeated['orphan'] == 0
|
||||
assert repeated['missing'] == 0
|
||||
assert repeated['removed'] == 0
|
||||
assert repeated['marked_failed'] == 0
|
||||
|
||||
object_keys = {item.key for item in await storage.list('originals/')}
|
||||
assert correct_key in object_keys
|
||||
assert restored_key in object_keys
|
||||
assert changed_key in object_keys
|
||||
assert orphan_key not in object_keys
|
||||
async with sessions() as session:
|
||||
missing_row = await session.get(AttachmentRow, missing_id)
|
||||
assert missing_row is not None
|
||||
assert missing_row.status == 'failed'
|
||||
assert missing_row.error_code == MISSING_ERROR_CODE
|
||||
assert await session.scalar(
|
||||
select(MessageAttachmentRow.attachment_id).where(
|
||||
MessageAttachmentRow.message_id == message_id
|
||||
)
|
||||
) == correct_id
|
||||
|
||||
restored_row = await session.get(AttachmentRow, restored_id)
|
||||
assert restored_row is not None
|
||||
assert restored_row.status == 'ready'
|
||||
|
||||
changed_row = await session.get(AttachmentRow, changed_id)
|
||||
assert changed_row is not None
|
||||
assert changed_row.status == 'ready'
|
||||
|
||||
# An object younger than the grace period survives a real apply run.
|
||||
await storage.put(fresh_key, BytesIO(b'x'), 1, 'application/octet-stream')
|
||||
fresh_apply = await reconcile(
|
||||
sessions,
|
||||
storage,
|
||||
apply=True,
|
||||
grace_seconds=60,
|
||||
now=datetime.now(timezone.utc),
|
||||
output=lambda _line: None,
|
||||
)
|
||||
assert fresh_apply['fresh_protected'] == 1
|
||||
assert fresh_apply['removed'] == 0
|
||||
assert await storage.exists(fresh_key)
|
||||
await storage.remove(fresh_key)
|
||||
|
||||
# Metadata created after the initial snapshots also protects an old
|
||||
# object from the destructive orphan action.
|
||||
await storage.put(appeared_key, BytesIO(b'x'), 1, 'application/octet-stream')
|
||||
|
||||
async def create_metadata_after_snapshot():
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
session.add(AttachmentRow(
|
||||
id=appeared_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=user_id,
|
||||
filename='appeared.png',
|
||||
object_key=appeared_key,
|
||||
content_type='image/png',
|
||||
size_bytes=1,
|
||||
sha256='0' * 64,
|
||||
width=1,
|
||||
height=1,
|
||||
status='ready',
|
||||
error_code=None,
|
||||
created_at=old,
|
||||
updated_at=old,
|
||||
))
|
||||
|
||||
appeared_apply = await reconcile(
|
||||
sessions,
|
||||
AfterListStorage(storage, create_metadata_after_snapshot),
|
||||
apply=True,
|
||||
grace_seconds=60,
|
||||
now=datetime.now(timezone.utc) + timedelta(seconds=61),
|
||||
output=lambda _line: None,
|
||||
)
|
||||
assert appeared_apply['orphan'] == 1
|
||||
assert appeared_apply['removed'] == 0
|
||||
assert appeared_apply['race_protected'] == 1
|
||||
assert await storage.exists(appeared_key)
|
||||
finally:
|
||||
await storage.remove(correct_key)
|
||||
await storage.remove(orphan_key)
|
||||
await storage.remove(restored_key)
|
||||
await storage.remove(changed_key)
|
||||
await storage.remove(appeared_key)
|
||||
await storage.remove(fresh_key)
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
chat = await session.get(ChatRow, chat_id)
|
||||
if chat is not None:
|
||||
await session.delete(chat)
|
||||
await session.flush()
|
||||
user = await session.get(UserRow, user_id)
|
||||
if user is not None:
|
||||
await session.delete(user)
|
||||
await engine.dispose()
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Простой профиль параллельных загрузок для измерения RAM контейнеров API."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from client import Client, ROOT, arguments, run
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--uploads', type=int, default=30)
|
||||
|
||||
args = arguments(
|
||||
'Parallel attachment uploads',
|
||||
configure,
|
||||
)
|
||||
if args.concurrency < 10:
|
||||
raise ValueError('concurrency must be at least 10 for lab4 evidence')
|
||||
if args.uploads < args.concurrency:
|
||||
raise ValueError('uploads must be greater than or equal to concurrency')
|
||||
client = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
def upload(index):
|
||||
local = Client(args.base_url, args.ca_file, timeout=30)
|
||||
started = time.monotonic()
|
||||
response = local.call(
|
||||
'POST',
|
||||
f'/api/v1/chats/{chat["id"]}/attachments?filename=load-{index}-{uuid.uuid4().hex}.png',
|
||||
201,
|
||||
actor=alice,
|
||||
raw=content,
|
||||
headers={'Content-Type': 'image/png'},
|
||||
schema='Attachment',
|
||||
)
|
||||
return response['headers']['X-Instance-Id'], time.monotonic() - started
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
results = list(pool.map(upload, range(args.uploads)))
|
||||
duration = time.monotonic() - started
|
||||
latencies = sorted(item[1] for item in results)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
'concurrency': args.concurrency,
|
||||
'file_size_bytes': len(content),
|
||||
'successful_uploads': len(results),
|
||||
'instances': len({item[0] for item in results}),
|
||||
'duration_seconds': round(duration, 3),
|
||||
'max_latency_seconds': round(latencies[-1], 3),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user