4 Commits
79 changed files with 4506 additions and 26 deletions
+13 -9
View File
@@ -4,17 +4,21 @@ APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
# Synthetic local credentials: replace them in your untracked .env.
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change-me-local
POSTGRES_DB=messenger
DATABASE_URL=postgresql+asyncpg://messenger:change-me-local@postgres:5432/messenger
REDIS_URL=redis://redis:6379/0
MINIO_ROOT_USER=local-minio-admin
MINIO_ROOT_PASSWORD=change-me-minio-root
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_ACCESS_KEY_ID=messenger-api
S3_SECRET_ACCESS_KEY=change-me-minio-api
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
LOG_LEVEL=info
+17
View File
@@ -0,0 +1,17 @@
FROM python:3.13.15-slim
WORKDIR /app
COPY requirements.txt .
RUN python -m pip install -r requirements.txt \
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app
COPY app ./app
COPY alembic ./alembic
COPY alembic.ini .
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=3).read()"]
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"]
+15
View File
@@ -0,0 +1,15 @@
FROM python:3.13.15-slim
WORKDIR /tests
COPY requirements.txt requirements-dev.txt ./
RUN python -m pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt \
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin tests
COPY lab.json .
COPY app ./app
COPY contracts ./contracts
COPY tests ./tests
RUN chown -R tests:tests /tests
USER 10001:10001
CMD ["python", "-m", "pytest", "-q"]
+20 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test test-attachments load load-attachments test-persistence test-failover test-attachment-failover reconcile-dry reconcile-apply benchmark-reconciler
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -18,9 +18,15 @@ validate:
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachments:
BASE_URL="$(BASE_URL)" $(PYTHON) -m pytest -q tests/test_attachments.py
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load-attachments:
$(PYTHON) tests/upload_load.py --base-url "$(BASE_URL)" --concurrency 10 --uploads 30 $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
@@ -28,3 +34,16 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachment-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/attachment_failover.py --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
reconcile-dry:
docker compose exec -T api python -m app.tools.reconcile_attachments --dry-run
reconcile-apply:
docker compose exec -T api python -m app.tools.reconcile_attachments --apply
benchmark-reconciler:
./scripts/benchmark-reconciler.sh
+251 -16
View File
@@ -1,37 +1,272 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Сергей Возжин, БПИ2301.
- Целевая оценка: 5; выбран трек 2 — согласованность PostgreSQL и S3.
- Commit/PR: TODO после публикации.
- Использованные источники: контракт репозитория, документация Docker Compose, Chainguard Images, MinIO SDK и Pillow.
- Библиотеки: FastAPI, Uvicorn, SQLAlchemy, asyncpg, Alembic, redis-py, MinIO Python SDK, Pillow, Argon2, pytest.
- Помощь ИИ: OpenAI Codex использовался для плана, реализации, тестов и оформления отчёта; результат проверен исполняемыми тестами.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Engine с Compose v2, Python 3.9+ для публичных тестов и свободный host-порт 8080. Проверенный стенд: Apple M4 Pro, 12 CPU cores, 24 GB RAM, Docker Desktop; API на Python 3.13.15.
Закреплённые компоненты:
- `python:3.13.15-slim`;
- `postgres:17.4-alpine`;
- `redis:7.4.2-alpine`;
- `haproxy:3.2.26-alpine`;
- Chainguard MinIO `RELEASE.2026-10-23T20-06-51Z`, OCI index digest `sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597`;
- Chainguard MinIO Client, вариант `latest-dev`, OCI index digest `sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805` (`mc --version` сообщает upstream build marker `DEVELOPMENT.GOGET`).
Готовые образы MinIO и `mc` скачиваются из реестра Chainguard `cgr.dev`; в Compose оба образа закреплены по OCI digest. Сервис `s3` использует минимальный образ `minio` и в учебном стенде запускается с `user: "0"`, чтобы иметь доступ к Docker volume `/data`; его порты на host не публикуются. Для одноразового `s3-bootstrap` используется образ `minio-client` с shell и `sed`, необходимыми для создания bucket, отдельного пользователя API и bucket-scoped policy.
Создать локальную конфигурацию и заменить синтетические пароли, не добавляя `.env` в Git:
```sh
cp .env.example .env
```
`S3_BUCKET` допускает изменение: bootstrap при каждом запуске заново применяет ограниченную
policy с текущим именем bucket. `MINIO_ROOT_*` получает только MinIO и bootstrap; API
получает отдельные `S3_ACCESS_KEY_ID` и `S3_SECRET_ACCESS_KEY`. Access key и secret
считаются параметрами первоначальной инициализации пользователя MinIO; их ротация требует
явной операторской процедуры, а не изменения `.env` на работающем volume.
Проверить конфигурацию, собрать образы, применить миграции и дождаться готовности:
```sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait
docker compose --env-file .env ps
```
`migrate` выполняет `alembic upgrade head` отдельно и однократно. `s3-bootstrap` идемпотентно создаёт приватный bucket, отдельного пользователя API и bucket-scoped policy. Две API-реплики стартуют только после миграции, Redis и успешного bootstrap. Единственная публичная точка входа — `http://localhost:8080`; API, PostgreSQL, Redis, S3 API и административная консоль MinIO наружу не публикуются.
Основные проверки:
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make check
make validate PYTHON=.venv/bin/python
make test
make test-attachments PYTHON=.venv/bin/python
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
docker compose --env-file .env up -d --wait --wait-timeout 60 api lb
make test-attachment-failover ACTION_SCRIPT=scripts/stop-one.sh
docker compose --env-file .env up -d --wait --wait-timeout 60 api lb
scripts/check-private-bucket.sh
scripts/check-s3-outage.sh
.venv/bin/python scripts/measure-upload-memory.py --concurrency 10 --uploads 30
make reconcile-dry
make benchmark-reconciler
```
Исправления следует сначала просмотреть через `make reconcile-dry`. Изменяющий данные
`make reconcile-apply` запускается оператором только после проверки dry-run.
`make benchmark-reconciler` по умолчанию создаёт 100, 1000 и 10000 изолированных пар
metadata/object, выполняет по три dry-run для каждого размера и удаляет тестовые данные.
Набор и число повторов меняются через `SIZES="..."` и `RUNS=...`; старый одиночный
параметр `SIZE=...` также поддерживается.
Каждый failover-тест намеренно оставляет выбранную API-реплику остановленной, поэтому между сценариями и после последнего сценария она явно восстанавливается указанной командой `docker compose up`.
Безопасная остановка с сохранением данных:
```sh
docker compose --env-file .env down
```
Намеренное необратимое удаление учебных данных выполняет только оператор:
```sh
docker compose --env-file .env down -v
```
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
HTTP client
|
v
HAProxy :8080
|
+-- round-robin --> API replica 1 --+--> PostgreSQL: metadata
| +--> Redis: sessions
+-- round-robin --> API replica 2 --+--> private MinIO: original bytes
```
Из lab3 сохранены две взаимозаменяемые API-реплики, HAProxy, общие PostgreSQL/Redis, Bearer-сессии, `X-Instance-Id`, общая идемпотентность сообщений, readiness и failover hooks.
В lab4 добавлены:
- MinIO и persistent `s3_data` volume;
- таблицы `attachments` и `message_attachments`;
- синхронный binary upload PNG/JPEG;
- авторизованные metadata/download endpoints;
- `attachment_ids` в сообщениях и в области сравнения идемпотентного body;
- компенсационное удаление объекта и безопасный cleanup старых orphan-объектов;
- повторяемый reconciler метаданных PostgreSQL и оригиналов S3.
PostgreSQL является источником истины для метаданных. В БД не хранятся bytes изображения. MinIO хранит оригинал по серверному ключу `originals/<UUID>`. Пользовательский `filename` хранится только как метаданные и не используется в object key или пути файловой системы.
### Upload и лимиты
API сначала проверяет сессию, доступ к чату, filename и заявленный MIME. Тело читается чанками в `SpooledTemporaryFile`: до 1 MiB находится в памяти, затем временно сбрасывается на ephemeral disk API. Как только суммарный размер становится больше 10 MiB, запрос отклоняется с 413; превысивший лимит чанк в spool не записывается. Постоянной локальной копии нет.
Pillow определяет реальный PNG/JPEG, проверяет совпадение MIME, размеры до декодирования пикселей и целостность изображения через `verify()`. Максимум — 10 485 760 bytes и 40 000 000 пикселей. Пустой, повреждённый или чужой формат возвращает 415; превышение лимита bytes/pixels — 413.
S3-вызовы выполняются вне event loop. HTTP client S3 имеет ограниченные connect/read timeout и retry. Readiness проверяет PostgreSQL, Redis и существование bucket.
### Транзакционная граница PostgreSQL/S3
Атомарной транзакции между S3 и PostgreSQL нет. Для синхронного `201 ready` выбран порядок:
1. Полностью проверить ограниченный файл.
2. Выполнить S3 PUT.
3. Выполнить транзакцию INSERT метаданных.
4. Вернуть `201` только после успешного завершения обоих шагов.
Если PUT не удался, метаданные не создаются. Если PUT прошёл, а транзакция БД упала, API выполняет best-effort S3 DELETE. Unit-тест с внедрённой ошибкой БД подтверждает компенсацию.
Если компенсационное удаление также не удалось, остаётся orphan без метаданных. CLI находит такие объекты:
```sh
docker compose --env-file .env run --rm --no-deps api \
python -m app.tools.cleanup_orphans --dry-run --grace-seconds 3600
docker compose --env-file .env run --rm --no-deps api \
python -m app.tools.cleanup_orphans --apply --grace-seconds 3600
```
По умолчанию используется grace period один час; CLI запрещает значения меньше 60 секунд. Объекты с существующими метаданными и свежие orphan-объекты не удаляются. Сначала используется `--dry-run`, затем осознанный `--apply`.
### Reconciler PostgreSQL/S3 — трек 2
Гипотеза эксперимента: полный снимок PostgreSQL и listing MinIO дадут линейную
асимптотику `O(N)` по времени и памяти. На диапазоне 100–10000 согласованных пар рост
времени не должен становиться сверхлинейным; абсолютные значения зависят от Docker-host
и прогрева, поэтому для каждой точки используются три последовательных запуска и медиана.
Расширенный reconciler запускается внутри API-контейнера, где уже есть те же настройки и
ограниченные S3 credentials, что у приложения:
```sh
docker compose exec -T api \
python -m app.tools.reconcile_attachments --dry-run --grace-seconds 3600
docker compose exec -T api \
python -m app.tools.reconcile_attachments --apply --grace-seconds 3600
```
Он читает снимок всех `attachments` и список `originals/`, после чего различает:
- `orphan`: объект S3 есть, но ни одной строки с таким `object_key` нет;
- `missing`: строка в состоянии `ready` есть, а оригинала S3 нет;
- корректную пару: присутствуют обе стороны;
- свежее расхождение: возраст меньше grace period, изменять его нельзя.
В `--dry-run` выводятся `WOULD_DELETE_ORPHAN` и `WOULD_MARK_MISSING`, но записи и объекты
не меняются. В `--apply` старый orphan удаляется, а missing-метаданные сохраняются и
переходят в `status=failed`, `error_code=original_missing`. Сообщение и связь с вложением
не удаляются. Повторный `--apply` ничего не делает: orphan уже отсутствует, а `failed`
больше не считается ожидающим оригинал состоянием. При этом строка `failed` по-прежнему
защищает внезапно появившийся объект с тем же ключом от удаления как orphan.
Grace period защищает upload, который ещё выполняется, и запрещён ниже 60 секунд.
Используемые сообщением вложения не имеют особого исключения: любой известный PostgreSQL
ключ защищён одинаково, поэтому корректный оригинал связанного сообщения не удаляется.
Content для `failed` возвращает 409, а привязать такое вложение к новому сообщению нельзя.
Начальные снимки могут устареть во время apply. Поэтому перед удалением orphan reconciler
повторно проверяет отсутствие `object_key` в PostgreSQL, перед маркировкой missing выполняет
`HEAD/stat` объекта, а SQL `UPDATE` дополнительно сравнивает `id`, `object_key`, `status` и
исходный `updated_at`. Гонки логируются как `SKIP_*` и учитываются в `race_protected`.
Общей транзакции между PostgreSQL и S3 всё равно нет: остаётся узкое TOCTOU-окно между
последней проверкой и действием. Полностью закрыть его можно только протоколом состояний/
lease или запретом параллельного восстановления на время apply.
Операции между двумя хранилищами всё ещё не образуют общей транзакции. Если apply
прервётся, уже выполненное безопасное удаление orphan не требует отката, условный SQL
UPDATE меняет только всё ещё `ready` строки, а повторный запуск завершает оставшуюся
работу.
### Доступ и идемпотентность
Метаданные и content доступны любому участнику чата. Неизвестный объект и запрос постороннего одинаково возвращают 404. Download идёт через API без redirect. Публичный ответ не раскрывает object key, endpoint или credentials.
Сообщение принимает 0–10 уникальных `attachment_ids`. Все вложения должны быть `ready` и относиться к тому же чату. Пустой текст допустим только с вложением. Сравнение идемпотентного запроса включает упорядоченный список attachment ID; отсутствие поля и `[]` считаются одинаковыми.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Offline/OpenAPI | `make check`; `make validate PYTHON=.venv/bin/python` | оба PASS | `evidence/results.md` |
| Публичный smoke | `make test` | `PASS: lab4 public API scenario` | `evidence/results.md` |
| Негативные сценарии | `make test-attachments PYTHON=.venv/bin/python` | `12 passed` | `tests/test_attachments.py`, `evidence/results.md` |
| Компенсация S3/БД | unit-тест с ошибкой repository | S3 DELETE вызван для загруженного ключа; `1 passed` | `tests/test_attachment_compensation.py` |
| Private bucket | `scripts/check-private-bucket.sh` | anonymous GET получил 403 | `evidence/results.md` |
| Недоступность S3 | `scripts/check-s3-outage.sh` | readiness вернул структурированный 503; стенд восстановлен | `evidence/results.md` |
| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | прежние token, metadata и bytes доступны после restart PostgreSQL/Redis/MinIO/API | `evidence/results.md` |
| Общий lab3 failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | 111 записей, 1 переходная ошибка, восстановление 2,333 с, файл сохранён | `evidence/results.md` |
| Точный attachment failover | `make test-attachment-failover ACTION_SCRIPT=scripts/stop-one.sh` | загрузившая реплика остановлена, download через другую за 3,912 с, SHA-256 совпал | `evidence/results.md` |
| Concurrency/RAM | `scripts/measure-upload-memory.py --concurrency 10 --uploads 30` | 30/30 upload, обе реплики, пики 68,50/68,33 MiB | `evidence/upload-memory.json` |
| Идемпотентный bootstrap | повторный запуск `docker compose up` после restart/failover | bootstrap повторно завершился с code 0 | `evidence/results.md` |
| Cleanup dry-run | `cleanup_orphans --dry-run --grace-seconds 60` | 164 известных объекта, удалено 0 | `evidence/results.md` |
| Reconciler fault injection | реальный PostgreSQL + MinIO: orphan, missing, fresh и три конкурентных изменения | apply исправляет только подтверждённые расхождения; свежие/изменённые данные защищены; повторный apply — 0 действий | `tests/test_reconciler_integration.py`, `evidence/results.md` |
| Полный тестовый стенд | изолированный Compose, все pytest | `17 passed` | `evidence/results.md` |
| Стоимость обхода | `SIZES="100 1000 10000" RUNS=3 make benchmark-reconciler` | медианы: 80,712 / 227,467 / 1733,545 мс; Python peak 2,62 / 4,21 / 8,08 MB | `evidence/results.md` |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Проверяемые ожидания:
- остановка загрузившей API-реплики не меняет файл и не лишает вторую реплику доступа;
- restart обязательных хранилищ с volumes сохраняет метаданные, bytes и сессию;
- недоступность S3 наблюдаема как 503, а не 500;
- 10 параллельных upload не приводят к неограниченному росту RAM.
Профиль upload: 30 запросов, concurrency 10, fixture 111 817 bytes, closed-loop, без retry. Получено 30 успешных upload за 0,382 с; максимальная latency 0,300 с; задействованы обе API-реплики. Пиковая наблюдаемая RAM: 68,50 MiB и 68,33 MiB при лимите 256 MiB на реплику.
Fault injection reconciler выполняется на настоящих PostgreSQL и MinIO. Тест создаёт
старую `ready` строку без объекта, объект без строки и корректное вложение, уже связанное
с сообщением. Дополнительно он восстанавливает missing-объект, изменяет `updated_at`
другой missing-строки и создаёт metadata для orphan уже после получения начальных
снимков, а также проверяет настоящий свежий orphan.
Управляемые часы имитируют истечение grace period без минутного ожидания. Apply исправляет
только подтверждённые расхождения; свежий объект и все три конкурентно изменённые
сущности защищены. Повторный apply не повторяет исправления, связь сообщения сохраняется.
Воспроизводимый benchmark для каждого размера удаляет прежнюю фикстуру, создаёт одинаковое
число строк `attachments` и объектов MinIO, выполняет три полных dry-run и удаляет данные.
| Пар metadata/object | Время, мс (три запуска) | Медиана, мс | Медианный peak Python allocations |
| ---: | --- | ---: | ---: |
| 100 | 78,847; 80,712; 82,781 | 80,712 | 2 620 657 bytes |
| 1 000 | 232,254; 227,467; 226,264 | 227,467 | 4 205 016 bytes |
| 10 000 | 1701,005; 1792,955; 1733,545 | 1733,545 | 8 075 853 bytes |
Измерение включает запрос PostgreSQL, listing MinIO и сравнение снимков; подготовка и
очистка в `elapsed_ms` не входят. Результат согласуется с гипотезой `O(N)`: после заметной
фиксированной стоимости время и память монотонно растут, на исследованном диапазоне нет
признаков сверхлинейного взрыва. Это не доказательство для production-масштаба: измеряется
Python `tracemalloc`, а не RSS контейнера, и полный снимок всё равно требует памяти `O(N)`.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Restart API + PostgreSQL + Redis + MinIO | token, metadata и bytes сохраняются | persistence PASS | потерь нет |
| Остановка API в общем failover | вторая реплика продолжает чтение/запись | 111 записей, 1 краткая ошибка | 2,333 с; потерь и дублей нет |
| Остановка загрузившей API | файл скачивается через другую | SHA-256 совпал, разные instance ID | 3,912 с; потерь нет |
| Остановка MinIO | readiness сообщает отказ зависимости | структурированный 503, после запуска readiness восстановился | потерь нет |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Стенд выдерживает остановку одной API-реплики и штатный restart контейнеров с сохранёнными volumes. Он не выдерживает потерю Docker-host, единственного HAProxy, PostgreSQL, Redis или MinIO volume. Несколько API-процессов не делают единственные хранилища высокодоступными.
Upload синхронный, поэтому клиент ждёт S3 и PostgreSQL. `SpooledTemporaryFile` ограничивает RAM одной загрузки, но временный disk API должен иметь достаточный объём и отдельную квоту в production. Reconciler загружает полный список объектов и строк БД в память, поэтому его стоимость O(N), а для большого bucket нужны страничный/потоковый обход, метрики, checkpoint и плановый запуск.
Chainguard-образы зафиксированы по digest, поэтому обновляются только явно. Стенд изолирован учебной сетью. Перед production-развёртыванием потребовались бы непривилегированный UID с корректными правами на volume, TLS, внешний secret manager, HA хранилищ, аудит и резервное копирование.
Presigned upload, worker, thumbnail и варианты изображений намеренно не реализованы: для выбранного трека 2 они не требуются.
+36
View File
@@ -0,0 +1,36 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = placeholder://configured-at-runtime
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
import asyncio
from logging.config import fileConfig
from alembic import context
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import async_engine_from_config
from app.config import load_settings
from app.db.postgres import Base
from app import models # noqa: F401 - imports ORM models into Base.metadata
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def _url() -> str:
return load_settings().database_url
def run_migrations_offline() -> None:
context.configure(url=_url(), target_metadata=target_metadata, literal_binds=True, dialect_opts={'paramstyle': 'named'})
with context.begin_transaction():
context.run_migrations()
def _run_migrations(connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def _run_migrations_async() -> None:
configuration = config.get_section(config.config_ini_section, {})
configuration['sqlalchemy.url'] = _url()
connectable = async_engine_from_config(configuration, prefix='sqlalchemy.', poolclass=pool.NullPool)
async with connectable.connect() as connection:
await connection.run_sync(_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(_run_migrations_async())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+22
View File
@@ -0,0 +1,22 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,64 @@
"""Initial PostgreSQL schema for lab 2.
Revision ID: 20261019_0001
Revises:
Create Date: 2026-10-19
"""
from alembic import op
import sqlalchemy as sa
revision = '20261019_0001'
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
'users',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('username', sa.String(length=32), nullable=False),
sa.Column('display_name', sa.String(length=100), nullable=False),
sa.Column('password_hash', sa.Text(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('username'),
)
op.create_table(
'chats',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('title', sa.String(length=100), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint('id'),
)
op.create_table(
'chat_members',
sa.Column('chat_id', sa.Uuid(), nullable=False),
sa.Column('user_id', sa.Uuid(), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('chat_id', 'user_id'),
)
op.create_index('chat_members_user_chat_idx', 'chat_members', ['user_id', 'chat_id'])
op.create_table(
'messages',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('chat_id', sa.Uuid(), nullable=False),
sa.Column('sender_id', sa.Uuid(), nullable=False),
sa.Column('text', sa.String(length=4000), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['sender_id'], ['users.id']),
sa.PrimaryKeyConstraint('id'),
)
op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id'])
def downgrade() -> None:
op.drop_index('messages_history_idx', table_name='messages')
op.drop_table('messages')
op.drop_index('chat_members_user_chat_idx', table_name='chat_members')
op.drop_table('chat_members')
op.drop_table('chats')
op.drop_table('users')
@@ -0,0 +1,35 @@
"""Add shared message idempotency for lab 3.
Revision ID: 20261023_0002
Revises: 20261019_0001
Create Date: 2026-10-23
"""
from alembic import op
import sqlalchemy as sa
revision = '20261023_0002'
down_revision = '20261019_0001'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column(
'messages',
sa.Column('idempotency_key', sa.String(length=128), nullable=True),
)
op.create_unique_constraint(
'messages_sender_chat_idempotency_key',
'messages',
['sender_id', 'chat_id', 'idempotency_key'],
)
def downgrade() -> None:
op.drop_constraint(
'messages_sender_chat_idempotency_key',
'messages',
type_='unique',
)
op.drop_column('messages', 'idempotency_key')
@@ -0,0 +1,69 @@
"""Add S3 attachment metadata and message links for lab 4.
Revision ID: 20261030_0003
Revises: 20261023_0002
Create Date: 2026-10-30
"""
from alembic import op
import sqlalchemy as sa
revision = '20261030_0003'
down_revision = '20261023_0002'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
'attachments',
sa.Column('id', sa.Uuid(), nullable=False),
sa.Column('chat_id', sa.Uuid(), nullable=False),
sa.Column('owner_id', sa.Uuid(), nullable=False),
sa.Column('filename', sa.String(length=255), nullable=False),
sa.Column('object_key', sa.String(length=255), nullable=False),
sa.Column('content_type', sa.String(length=32), nullable=False),
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
sa.Column('sha256', sa.String(length=64), nullable=False),
sa.Column('width', sa.Integer(), nullable=False),
sa.Column('height', sa.Integer(), nullable=False),
sa.Column('status', sa.String(length=16), nullable=False),
sa.Column('error_code', sa.String(length=64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['owner_id'], ['users.id']),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('object_key', name='attachments_object_key_key'),
)
op.create_index(
'attachments_chat_created_idx',
'attachments',
['chat_id', 'created_at', 'id'],
)
op.create_table(
'message_attachments',
sa.Column('message_id', sa.Uuid(), nullable=False),
sa.Column('attachment_id', sa.Uuid(), nullable=False),
sa.Column('position', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('message_id', 'attachment_id'),
sa.UniqueConstraint(
'message_id',
'position',
name='message_attachments_message_position_key',
),
)
op.create_index(
'message_attachments_attachment_idx',
'message_attachments',
['attachment_id'],
)
def downgrade() -> None:
op.drop_index('message_attachments_attachment_idx', table_name='message_attachments')
op.drop_table('message_attachments')
op.drop_index('attachments_chat_created_idx', table_name='attachments')
op.drop_table('attachments')
+1
View File
@@ -0,0 +1 @@
"""Мессенджер на PostgreSQL и Redis, лабораторная 2."""
+1
View File
@@ -0,0 +1 @@
"""HTTP-граница: FastAPI, коды состояния, формат ошибок, курсоры."""
+28
View File
@@ -0,0 +1,28 @@
import base64
import json
from uuid import UUID
from ..domain.errors import Invalid
from ..domain.models import Keyset
from .timefmt import from_rfc3339, to_rfc3339
MAX_LENGTH = 512
def encode(scope: str, position: Keyset) -> str:
payload = {'s': scope, 't': to_rfc3339(position.created_at), 'i': str(position.id)}
raw = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8')
return base64.urlsafe_b64encode(raw).decode('ascii').rstrip('=')
def decode(scope: str, value: str) -> Keyset:
if not value or len(value) > MAX_LENGTH:
raise Invalid('cursor недействителен для этого запроса')
try:
raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4))
payload = json.loads(raw)
if not isinstance(payload, dict) or payload.get('s') != scope:
raise ValueError('scope mismatch')
return Keyset(created_at=from_rfc3339(payload['t']), id=UUID(payload['i']))
except Exception:
raise Invalid('cursor недействителен для этого запроса') from None
+78
View File
@@ -0,0 +1,78 @@
from __future__ import annotations
from dataclasses import dataclass
from fastapi import Depends, Header, HTTPException, Request, status
from ..db.redis import RedisSessionStore
from ..domain.errors import Unauthorized
from ..domain.models import User
from ..services.chats import ChatService
from ..services.attachments import AttachmentService
from ..services.messages import MessageService
from ..services.users import UserService
_BEARER_CHALLENGE = 'Bearer realm="messenger"'
@dataclass(frozen=True, slots=True)
class Services:
attachments: AttachmentService
users: UserService
chats: ChatService
messages: MessageService
sessions: RedisSessionStore
def services(request: Request) -> Services:
return request.app.state.services
def user_service(container: Services = Depends(services)) -> UserService:
return container.users
def chat_service(container: Services = Depends(services)) -> ChatService:
return container.chats
def message_service(container: Services = Depends(services)) -> MessageService:
return container.messages
def attachment_service(container: Services = Depends(services)) -> AttachmentService:
return container.attachments
def session_store(container: Services = Depends(services)) -> RedisSessionStore:
return container.sessions
def _bearer_error() -> HTTPException:
return HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='требуется действующая Bearer-сессия',
headers={'WWW-Authenticate': _BEARER_CHALLENGE},
)
async def bearer_token(authorization: str | None = Header(None)) -> str:
if not authorization:
raise _bearer_error()
scheme, separator, token = authorization.partition(' ')
if scheme.lower() != 'bearer' or not separator or not token or token.strip() != token:
raise _bearer_error()
return token
async def current_user(
token: str = Depends(bearer_token),
container: Services = Depends(services),
) -> User:
user_id = await container.sessions.get_user_id(token)
if user_id is None:
raise _bearer_error()
try:
return await container.users.identify(user_id)
except Unauthorized:
raise _bearer_error() from None
+107
View File
@@ -0,0 +1,107 @@
import logging
from fastapi import FastAPI
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from sqlalchemy.exc import SQLAlchemyError
from starlette.exceptions import HTTPException as StarletteHTTPException
from ..domain.errors import (
Conflict,
DomainError,
Invalid,
NotFound,
PayloadTooLarge,
Unauthorized,
Unavailable,
UnsupportedMediaType,
)
logger = logging.getLogger('app.error')
CODE_BY_STATUS = {
400: 'invalid_request',
401: 'unauthorized',
404: 'not_found',
409: 'conflict',
413: 'payload_too_large',
415: 'unsupported_media_type',
429: 'rate_limited',
500: 'internal_error',
503: 'unavailable',
}
STATUS_BY_DOMAIN = {
Invalid: 400,
Unauthorized: 401,
NotFound: 404,
Conflict: 409,
PayloadTooLarge: 413,
UnsupportedMediaType: 415,
Unavailable: 503,
}
def code_for(status: int) -> str:
if status in CODE_BY_STATUS:
return CODE_BY_STATUS[status]
return 'invalid_request' if status < 500 else 'internal_error'
def error_response(status: int, message: str, request_id: str, code: str = '') -> JSONResponse:
return JSONResponse(
status_code=status,
content={'error': {'code': code or code_for(status), 'message': message, 'request_id': request_id}},
)
def _domain_status(error: DomainError) -> int:
for kind, status in STATUS_BY_DOMAIN.items():
if isinstance(error, kind):
return status
return 500
def _describe(error: RequestValidationError) -> str:
"""Краткое указание на пол, без stack trace и внутреннего устройства"""
for item in error.errors():
if item.get('type') == 'json_invalid':
return 'тело запроса не является корректным JSON'
location = '.'.join(str(part) for part in item.get('loc', ()) if part != 'body')
return f'некорректное поле {location}' if location else 'некорректный запрос'
return 'некорректный запрос'
def request_id(request) -> str:
return getattr(request.state, 'request_id', 'unknown')
def install(app: FastAPI) -> None:
@app.exception_handler(DomainError)
async def _on_domain(request, error: DomainError) -> JSONResponse:
return error_response(_domain_status(error), error.message, request_id(request))
@app.exception_handler(RequestValidationError)
async def _on_validation(request, error: RequestValidationError) -> JSONResponse:
return error_response(400, _describe(error), request_id(request))
@app.exception_handler(SQLAlchemyError)
async def _on_database(request, error: SQLAlchemyError) -> JSONResponse:
logger.warning('database error request_id=%s type=%s', request_id(request), type(error).__name__)
return error_response(503, 'база данных недоступна', request_id(request))
@app.exception_handler(OSError)
async def _on_storage_socket(request, error: OSError) -> JSONResponse:
logger.warning('storage socket error request_id=%s type=%s', request_id(request), type(error).__name__)
return error_response(503, 'обязательное хранилище недоступно', request_id(request))
@app.exception_handler(StarletteHTTPException)
async def _on_http(request, error: StarletteHTTPException) -> JSONResponse:
message = error.detail if isinstance(error.detail, str) and error.detail else 'ошибка запроса'
response = error_response(error.status_code, message, request_id(request))
for name, value in (error.headers or {}).items():
response.headers[name] = value
return response
__all__ = ['code_for', 'error_response', 'install']
+43
View File
@@ -0,0 +1,43 @@
import logging
import os
import socket
from uuid import uuid4
from starlette.datastructures import MutableHeaders
from .errors import error_response
logger = logging.getLogger('app')
class RequestContext:
def __init__(self, app) -> None:
self.app = app
self.instance_id = os.getenv('INSTANCE_ID') or socket.gethostname()
async def __call__(self, scope, receive, send) -> None:
if scope['type'] != 'http':
await self.app(scope, receive, send)
return
request_id = uuid4().hex
scope.setdefault('state', {})['request_id'] = request_id
started = False
async def send_wrapper(message) -> None:
nonlocal started
if message['type'] == 'http.response.start':
started = True
headers = MutableHeaders(scope=message)
headers['X-Request-Id'] = request_id
headers['X-Instance-Id'] = self.instance_id
await send(message)
try:
await self.app(scope, receive, send_wrapper)
except Exception:
logger.exception(f'unhandled error request_id={request_id} path={scope.get("path")}')
if started:
raise
response = error_response(500, 'внутренняя ошибка', request_id)
await response(scope, receive, send_wrapper)
+3
View File
@@ -0,0 +1,3 @@
from . import auth, chats, health, messages, users
__all__ = ['auth', 'chats', 'health', 'messages', 'users']
+66
View File
@@ -0,0 +1,66 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Depends, Header, Query, Request
from fastapi.responses import StreamingResponse
from ...domain.models import User
from ...services.attachments import AttachmentService
from ..deps import attachment_service, current_user
from ..schemas import AttachmentOut
router = APIRouter(tags=['attachments'])
@router.post(
'/api/v1/chats/{chat_id}/attachments',
status_code=201,
response_model=AttachmentOut,
)
async def upload(
chat_id: UUID,
request: Request,
filename: str = Query(..., min_length=1, max_length=255),
content_type: Annotated[str | None, Header(alias='Content-Type')] = None,
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> AttachmentOut:
attachment = await attachments.upload(
actor,
chat_id,
filename,
content_type or '',
request.stream(),
)
return AttachmentOut.from_domain(attachment)
@router.get('/api/v1/attachments/{attachment_id}', response_model=AttachmentOut)
async def metadata(
attachment_id: UUID,
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> AttachmentOut:
return AttachmentOut.from_domain(await attachments.get(actor, attachment_id))
@router.get('/api/v1/attachments/{attachment_id}/content')
async def content(
attachment_id: UUID,
variant: str = Query('original'),
actor: User = Depends(current_user),
attachments: AttachmentService = Depends(attachment_service),
) -> StreamingResponse:
if variant != 'original':
from ...domain.errors import NotFound
raise NotFound('вариант вложения не найден')
attachment, stream = await attachments.content(actor, attachment_id)
return StreamingResponse(
stream.chunks(),
media_type=attachment.content_type,
headers={
'Content-Length': str(attachment.size_bytes),
'Content-Disposition': f'inline; filename="attachment-{attachment.id}"',
},
)
+71
View File
@@ -0,0 +1,71 @@
from __future__ import annotations
import base64
from binascii import Error as Base64Error
from fastapi import APIRouter, Depends, Header, HTTPException, status
from ...auth.tokens import create_token
from ...db.redis import RedisSessionStore
from ...domain.models import User
from ...services.users import UserService
from ..deps import bearer_token, current_user, session_store, user_service
from ..schemas import SessionOut, UserOut
from ..timefmt import to_rfc3339
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
_BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
def _unauthorized_basic() -> HTTPException:
return HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail='неверные учётные данные',
headers={'WWW-Authenticate': _BASIC_CHALLENGE},
)
def _parse_basic(authorization: str | None) -> tuple[str, str]:
if not authorization:
raise _unauthorized_basic()
scheme, separator, encoded = authorization.partition(' ')
if scheme.lower() != 'basic' or not separator or not encoded:
raise _unauthorized_basic()
try:
decoded = base64.b64decode(encoded, validate=True).decode('utf-8')
except (Base64Error, UnicodeDecodeError):
raise _unauthorized_basic() from None
username, separator, password = decoded.partition(':')
if not separator or not username:
raise _unauthorized_basic()
return username, password
@router.post('', status_code=status.HTTP_201_CREATED, response_model=SessionOut)
async def create_session(
authorization: str | None = Header(None),
users: UserService = Depends(user_service),
sessions: RedisSessionStore = Depends(session_store),
) -> SessionOut:
username, password = _parse_basic(authorization)
user = await users.authenticate(username, password)
if user is None:
raise _unauthorized_basic()
token = create_token()
expires_at = await sessions.create(user.id, token)
return SessionOut(
token=token,
token_type='Bearer',
expires_at=to_rfc3339(expires_at),
user=UserOut.from_domain(user),
)
@router.delete('/current', status_code=status.HTTP_204_NO_CONTENT, response_model=None)
async def revoke_session(
token: str = Depends(bearer_token),
_: User = Depends(current_user),
sessions: RedisSessionStore = Depends(session_store),
) -> None:
await sessions.revoke(token)
+49
View File
@@ -0,0 +1,49 @@
from typing import Optional
from uuid import UUID
from fastapi import APIRouter, Depends, Query
from ...domain.models import User
from ...services.chats import ChatService
from .. import cursors
from ..deps import chat_service, current_user
from ..schemas import (
LIMIT_DEFAULT,
LIMIT_MAX,
LIMIT_MIN,
ChatOut,
ChatPageOut,
CreateChat,
)
router = APIRouter(prefix='/api/v1/chats', tags=['chats'])
@router.post('', status_code=201, response_model=ChatOut)
async def create(
body: CreateChat,
actor: User = Depends(current_user),
chats: ChatService = Depends(chat_service),
) -> ChatOut:
return ChatOut.from_domain(await chats.create(actor, body.title, body.member_ids))
@router.get('', response_model=ChatPageOut)
async def page(
actor: User = Depends(current_user),
chats: ChatService = Depends(chat_service),
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
) -> ChatPageOut:
scope = f'chats:{actor.id}'
after = cursors.decode(scope, cursor) if cursor is not None else None
return ChatPageOut.from_domain(await chats.page(actor, after, limit), scope)
@router.get('/{chat_id}', response_model=ChatOut)
async def get(
chat_id: UUID,
actor: User = Depends(current_user),
chats: ChatService = Depends(chat_service),
) -> ChatOut:
return ChatOut.from_domain(await chats.get(actor, chat_id))
+29
View File
@@ -0,0 +1,29 @@
from fastapi import APIRouter, HTTPException, Request
from redis.exceptions import RedisError
from sqlalchemy import text
from sqlalchemy.exc import SQLAlchemyError
from ...domain.errors import Unavailable
from ..schemas import HealthOut
router = APIRouter(tags=['health'])
@router.get('/health/live', response_model=HealthOut)
async def live() -> HealthOut:
return HealthOut(status='ok')
@router.get('/health/ready', response_model=HealthOut)
async def ready(request: Request) -> HealthOut:
if not request.app.state.ready:
raise HTTPException(status_code=503, detail='приложение не готово принимать трафик')
try:
async with request.app.state.engine.connect() as connection:
await connection.execute(text('SELECT 1'))
await request.app.state.redis.ping()
await request.app.state.storage.ping()
except (SQLAlchemyError, RedisError, Unavailable, OSError):
raise HTTPException(status_code=503, detail='обязательное хранилище недоступно') from None
return HealthOut(status='ok')
+53
View File
@@ -0,0 +1,53 @@
from typing import Annotated, Optional
from uuid import UUID
from fastapi import APIRouter, Depends, Header, Query
from ...domain.models import User
from ...services.messages import MessageService
from .. import cursors
from ..deps import current_user, message_service
from ..schemas import (
LIMIT_DEFAULT,
LIMIT_MAX,
LIMIT_MIN,
CreateMessage,
MessageOut,
MessagePageOut,
)
router = APIRouter(prefix='/api/v1/chats/{chat_id}/messages', tags=['messages'])
@router.post('', status_code=201, response_model=MessageOut)
async def send(
chat_id: UUID,
body: CreateMessage,
idempotency_key: Annotated[
str,
Header(
alias='Idempotency-Key',
min_length=1,
max_length=128,
pattern=r'^[A-Za-z0-9._:-]+$',
),
],
actor: User = Depends(current_user),
messages: MessageService = Depends(message_service),
) -> MessageOut:
return MessageOut.from_domain(
await messages.send(actor, chat_id, body.text, body.attachment_ids, idempotency_key)
)
@router.get('', response_model=MessagePageOut)
async def page(
chat_id: UUID,
actor: User = Depends(current_user),
messages: MessageService = Depends(message_service),
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
) -> MessagePageOut:
scope = f'messages:{chat_id}'
after = cursors.decode(scope, cursor) if cursor is not None else None
return MessagePageOut.from_domain(await messages.history(actor, chat_id, after, limit), scope)
+18
View File
@@ -0,0 +1,18 @@
from fastapi import APIRouter, Depends
from ...domain.models import User
from ...services.users import UserService
from ..deps import current_user, user_service
from ..schemas import CreateUser, UserOut
router = APIRouter(prefix='/api/v1', tags=['users'])
@router.post('/users', status_code=201, response_model=UserOut)
async def register(body: CreateUser, users: UserService = Depends(user_service)) -> UserOut:
return UserOut.from_domain(await users.register(body.username, body.display_name, body.password))
@router.get('/users/me', response_model=UserOut)
async def me(actor: User = Depends(current_user)) -> UserOut:
return UserOut.from_domain(actor)
+190
View File
@@ -0,0 +1,190 @@
from typing import Optional
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from ..domain.models import Attachment, Chat, Message, Page, User
from . import cursors
from .timefmt import to_rfc3339
NAME_MAX = 100
TEXT_MAX = 4000
MEMBERS_MAX = 100
LIMIT_MIN = 1
LIMIT_MAX = 100
LIMIT_DEFAULT = 50
def not_blank(value: str) -> str:
if not value.strip():
raise ValueError('строка не может состоять только из пробельных символов')
return value
class Strict(BaseModel):
model_config = ConfigDict(extra='forbid')
class HealthOut(Strict):
status: str = 'ok'
class CreateUser(Strict):
username: str = Field(min_length=3, max_length=32, pattern=r'^[a-z0-9_]{3,32}$')
display_name: str = Field(min_length=1, max_length=NAME_MAX)
password: str = Field(min_length=12, max_length=128)
@field_validator('display_name')
@classmethod
def _display_name(cls, value: str) -> str:
return not_blank(value)
class UserOut(Strict):
id: UUID
username: str
display_name: str
created_at: str
@classmethod
def from_domain(cls, user: User) -> 'UserOut':
return cls(
id=user.id,
username=user.username,
display_name=user.display_name,
created_at=to_rfc3339(user.created_at),
)
class SessionOut(Strict):
token: str = Field(min_length=32)
token_type: str = 'Bearer'
expires_at: str
user: UserOut
class CreateChat(Strict):
title: str = Field(min_length=1, max_length=NAME_MAX)
member_ids: list[UUID] = Field(min_length=1, max_length=MEMBERS_MAX)
@field_validator('title')
@classmethod
def _title(cls, value: str) -> str:
return not_blank(value)
@field_validator('member_ids')
@classmethod
def _members_unique(cls, value: list[UUID]) -> list[UUID]:
if len(set(value)) != len(value):
raise ValueError('member_ids не должен содержать дубликаты')
return value
class ChatOut(Strict):
id: UUID
title: str
member_ids: list[UUID]
created_at: str
@classmethod
def from_domain(cls, chat: Chat) -> 'ChatOut':
return cls(
id=chat.id,
title=chat.title,
member_ids=list(chat.member_ids),
created_at=to_rfc3339(chat.created_at),
)
class CreateMessage(Strict):
text: str = Field(max_length=TEXT_MAX)
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
@field_validator('attachment_ids')
@classmethod
def _attachments_unique(cls, value: list[UUID]) -> list[UUID]:
if len(set(value)) != len(value):
raise ValueError('attachment_ids не должен содержать дубликаты')
return value
@model_validator(mode='after')
def _content_present(self) -> 'CreateMessage':
if not self.text.strip() and not self.attachment_ids:
raise ValueError('сообщение должно содержать текст или вложение')
return self
class MessageOut(Strict):
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: str
attachment_ids: list[UUID]
@classmethod
def from_domain(cls, message: Message) -> 'MessageOut':
return cls(
id=message.id,
chat_id=message.chat_id,
sender_id=message.sender_id,
text=message.text,
created_at=to_rfc3339(message.created_at),
attachment_ids=list(message.attachment_ids),
)
class AttachmentOut(Strict):
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: str
variants: list = Field(default_factory=list)
error_code: str | None
created_at: str
updated_at: str
@classmethod
def from_domain(cls, attachment: Attachment) -> 'AttachmentOut':
return cls(
id=attachment.id,
chat_id=attachment.chat_id,
owner_id=attachment.owner_id,
filename=attachment.filename,
content_type=attachment.content_type,
size_bytes=attachment.size_bytes,
sha256=attachment.sha256,
status=attachment.status,
variants=[],
error_code=attachment.error_code,
created_at=to_rfc3339(attachment.created_at),
updated_at=to_rfc3339(attachment.updated_at),
)
class ChatPageOut(Strict):
items: list[ChatOut]
next_cursor: Optional[str]
@classmethod
def from_domain(cls, page: Page, scope: str) -> 'ChatPageOut':
return cls(
items=[ChatOut.from_domain(chat) for chat in page.items],
next_cursor=cursors.encode(scope, page.next) if page.next else None,
)
class MessagePageOut(Strict):
items: list[MessageOut]
next_cursor: Optional[str]
@classmethod
def from_domain(cls, page: Page, scope: str) -> 'MessagePageOut':
return cls(
items=[MessageOut.from_domain(message) for message in page.items],
next_cursor=cursors.encode(scope, page.next) if page.next else None,
)
+11
View File
@@ -0,0 +1,11 @@
from datetime import datetime, timezone
def to_rfc3339(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def from_rfc3339(value: str) -> datetime:
if not isinstance(value, str) or not value.endswith('Z'):
raise ValueError('timestamp must end with Z')
return datetime.fromisoformat(value[:-1]).replace(tzinfo=timezone.utc)
+4
View File
@@ -0,0 +1,4 @@
from .passwords import hash_password, password_matches
from .tokens import create_token
__all__ = ['create_token', 'hash_password', 'password_matches']
+15
View File
@@ -0,0 +1,15 @@
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError, VerifyMismatchError
_hasher = PasswordHasher()
def hash_password(password: str) -> str:
return _hasher.hash(password)
def password_matches(stored_hash: str, password: str) -> bool:
try:
return _hasher.verify(stored_hash, password)
except (VerifyMismatchError, VerificationError):
return False
+11
View File
@@ -0,0 +1,11 @@
import hashlib
import secrets
def create_token() -> str:
return secrets.token_urlsafe(32)
def session_key(token: str) -> str:
digest = hashlib.sha256(token.encode('utf-8')).hexdigest()
return f'session:{digest}'
+3
View File
@@ -0,0 +1,3 @@
from .settings import Settings, load_settings
__all__ = ['Settings', 'load_settings']
+54
View File
@@ -0,0 +1,54 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
def _read_value(name: str, file_name: str) -> str:
value = os.getenv(name)
if value:
return value
path = os.getenv(file_name)
if path:
return Path(path).read_text(encoding='utf-8').strip()
raise RuntimeError(f'не задана настройка {name} или {file_name}')
@dataclass(frozen=True, slots=True)
class Settings:
database_url: str
redis_url: str
session_ttl_seconds: int
s3_endpoint: str
s3_region: str
s3_bucket: str
s3_access_key_id: str
s3_secret_access_key: str
max_upload_bytes: int
max_image_pixels: int
def _positive_int(name: str, default: str) -> int:
try:
value = int(os.getenv(name, default))
except ValueError as exc:
raise RuntimeError(f'{name} должен быть целым числом') from exc
if value <= 0:
raise RuntimeError(f'{name} должен быть больше нуля')
return value
def load_settings() -> Settings:
return Settings(
database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'),
redis_url=_read_value('REDIS_URL', 'CACHE_URL_FILE'),
session_ttl_seconds=_positive_int('SESSION_TTL_SECONDS', '3600'),
s3_endpoint=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
s3_region=os.getenv('S3_REGION', 'us-east-1'),
s3_bucket=os.getenv('S3_BUCKET', 'messenger-attachments'),
s3_access_key_id=_read_value('S3_ACCESS_KEY_ID', 'S3_ACCESS_KEY_ID_FILE'),
s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'),
max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'),
max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'),
)
+4
View File
@@ -0,0 +1,4 @@
from .postgres import Base, create_engine, create_session_factory
from .redis import RedisSessionStore, create_redis
__all__ = ['Base', 'RedisSessionStore', 'create_engine', 'create_redis', 'create_session_factory']
+23
View File
@@ -0,0 +1,23 @@
from __future__ import annotations
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import DeclarativeBase
class Base(DeclarativeBase):
pass
def create_engine(url: str) -> AsyncEngine:
return create_async_engine(
url,
pool_size=5,
max_overflow=5,
pool_timeout=5,
pool_pre_ping=True,
connect_args={'server_settings': {'statement_timeout': '5000'}},
)
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
return async_sessionmaker(engine, expire_on_commit=False)
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from uuid import UUID
from redis.asyncio import Redis
from redis.exceptions import RedisError
from ..auth.tokens import session_key
from ..domain.errors import Unavailable
def create_redis(url: str) -> Redis:
return Redis.from_url(url, decode_responses=True, socket_connect_timeout=3, socket_timeout=3)
class RedisSessionStore:
def __init__(self, client: Redis, ttl_seconds: int) -> None:
self._client = client
self._ttl_seconds = ttl_seconds
async def create(self, user_id: UUID, token: str) -> datetime:
try:
await self._client.set(session_key(token), str(user_id), ex=self._ttl_seconds)
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
return datetime.now(timezone.utc) + timedelta(seconds=self._ttl_seconds)
async def get_user_id(self, token: str) -> UUID | None:
try:
value = await self._client.get(session_key(token))
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
if value is None:
return None
try:
return UUID(value)
except ValueError:
return None
async def revoke(self, token: str) -> None:
try:
await self._client.delete(session_key(token))
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
async def ping(self) -> None:
try:
await self._client.ping()
except RedisError as exc:
raise Unavailable('хранилище сессий недоступно') from exc
View File
+37
View File
@@ -0,0 +1,37 @@
from __future__ import annotations
class DomainError(Exception):
"""Сообщение попадает клиенту, поэтому не содержит внутренних деталей."""
def __init__(self, message: str) -> None:
super().__init__(message)
self.message = message
class Invalid(DomainError):
"""Запрос нарушает правила предметной области."""
class Unauthorized(DomainError):
"""Нет действительной идентификации."""
class NotFound(DomainError):
"""Объекта нет либо он недоступен вызывающему: снаружи это неразличимо."""
class Conflict(DomainError):
"""Состояние не позволяет выполнить операцию."""
class Unavailable(DomainError):
"""Обязательная внешняя зависимость временно недоступна."""
class PayloadTooLarge(DomainError):
"""Тело запроса или декодированное содержимое превышает лимит."""
class UnsupportedMediaType(DomainError):
"""Формат содержимого не поддерживается или не соответствует MIME."""
+75
View File
@@ -0,0 +1,75 @@
from dataclasses import dataclass
from datetime import datetime
from typing import Optional
from uuid import UUID
@dataclass(frozen=True, slots=True)
class User:
id: UUID
username: str
display_name: str
created_at: datetime
@dataclass(frozen=True, slots=True)
class Chat:
id: UUID
title: str
member_ids: tuple[UUID, ...]
created_at: datetime
def has_member(self, user_id: UUID) -> bool:
return user_id in self.member_ids
@dataclass(frozen=True, slots=True)
class Message:
id: UUID
chat_id: UUID
sender_id: UUID
text: str
created_at: datetime
attachment_ids: tuple[UUID, ...] = ()
@dataclass(frozen=True, slots=True)
class Attachment:
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
object_key: str
content_type: str
size_bytes: int
sha256: str
width: int
height: int
status: str
error_code: str | None
created_at: datetime
updated_at: datetime
@dataclass(frozen=True, slots=True)
class Keyset:
created_at: datetime
id: UUID
@property
def order(self) -> tuple[datetime, int]:
return (self.created_at, self.id.int)
@classmethod
def from_item(cls, item) -> 'Keyset':
return cls(created_at=item.created_at, id=item.id)
def order_key(item) -> tuple[datetime, int]:
return (item.created_at, item.id.int)
@dataclass(frozen=True, slots=True)
class Page:
items: tuple
next: Optional[Keyset]
+112
View File
@@ -0,0 +1,112 @@
"""Сборка FastAPI приложения и подключение обязательных хранилищ."""
from __future__ import annotations
import logging
import os
import sys
from contextlib import asynccontextmanager
from fastapi import FastAPI
from sqlalchemy import text
from .api import errors
from .api.deps import Services
from .api.middleware import RequestContext
from .api.routers import attachments, auth, chats, health, messages, users
from .config import load_settings
from .db import RedisSessionStore, create_engine, create_redis, create_session_factory
from .repositories import AttachmentRepository, ChatRepository, MessageRepository, UserRepository
from .services.attachments import AttachmentService
from .services.chats import ChatService
from .services.messages import MessageService
from .services.users import UserService
from .storage import S3Storage
logger = logging.getLogger('app')
DESCRIPTION = 'PostgreSQL хранит метаданные, Redis — сессии, приватный S3 — оригиналы вложений.'
def configure_logging() -> None:
logging.basicConfig(
level=os.getenv('LOG_LEVEL', 'info').upper(),
stream=sys.stdout,
format='%(asctime)s %(levelname)s %(name)s %(message)s',
force=True,
)
def create_app() -> FastAPI:
configure_logging()
@asynccontextmanager
async def lifespan(application: FastAPI):
application.state.ready = False
settings = load_settings()
engine = create_engine(settings.database_url)
redis = create_redis(settings.redis_url)
storage = S3Storage(
endpoint=settings.s3_endpoint,
access_key=settings.s3_access_key_id,
secret_key=settings.s3_secret_access_key,
bucket=settings.s3_bucket,
region=settings.s3_region,
)
try:
async with engine.connect() as connection:
await connection.execute(text('SELECT 1'))
session_store = RedisSessionStore(redis, settings.session_ttl_seconds)
await session_store.ping()
await storage.ping()
session_factory = create_session_factory(engine)
user_repository = UserRepository(session_factory)
chat_repository = ChatRepository(session_factory)
message_repository = MessageRepository(session_factory)
attachment_repository = AttachmentRepository(session_factory)
application.state.services = Services(
attachments=AttachmentService(
attachment_repository,
storage,
settings.max_upload_bytes,
settings.max_image_pixels,
),
users=UserService(user_repository),
chats=ChatService(chat_repository),
messages=MessageService(message_repository),
sessions=session_store,
)
application.state.engine = engine
application.state.redis = redis
application.state.storage = storage
application.state.ready = True
logger.info('startup storage=postgres+s3 redis_sessions=true')
yield
finally:
application.state.ready = False
await redis.aclose()
await engine.dispose()
logger.info('shutdown complete')
api = FastAPI(
title='MTUSI Messenger. LR 4',
version='1.4.0',
description=DESCRIPTION,
lifespan=lifespan,
)
api.router.redirect_slashes = False
api.state.ready = False
api.add_middleware(RequestContext)
errors.install(api)
for router in (
health.router,
users.router,
auth.router,
chats.router,
messages.router,
attachments.router,
):
api.include_router(router)
return api
app = create_app()
+13
View File
@@ -0,0 +1,13 @@
from .attachment import AttachmentRow, MessageAttachmentRow
from .chat import ChatMemberRow, ChatRow
from .message import MessageRow
from .user import UserRow
__all__ = [
'AttachmentRow',
'ChatMemberRow',
'ChatRow',
'MessageAttachmentRow',
'MessageRow',
'UserRow',
]
+49
View File
@@ -0,0 +1,49 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class AttachmentRow(Base):
__tablename__ = 'attachments'
__table_args__ = (
Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'),
UniqueConstraint('object_key', name='attachments_object_key_key'),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
filename: Mapped[str] = mapped_column(String(255), nullable=False)
object_key: Mapped[str] = mapped_column(String(255), nullable=False)
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
width: Mapped[int] = mapped_column(Integer, nullable=False)
height: Mapped[int] = mapped_column(Integer, nullable=False)
status: Mapped[str] = mapped_column(String(16), nullable=False)
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class MessageAttachmentRow(Base):
__tablename__ = 'message_attachments'
__table_args__ = (
UniqueConstraint('message_id', 'position', name='message_attachments_message_position_key'),
Index('message_attachments_attachment_idx', 'attachment_id'),
)
message_id: Mapped[UUID] = mapped_column(
ForeignKey('messages.id', ondelete='CASCADE'), primary_key=True
)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
)
position: Mapped[int] = mapped_column(Integer, nullable=False)
+26
View File
@@ -0,0 +1,26 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import DateTime, ForeignKey, Index, String
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class ChatRow(Base):
__tablename__ = 'chats'
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
title: Mapped[str] = mapped_column(String(100), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
class ChatMemberRow(Base):
__tablename__ = 'chat_members'
__table_args__ = (Index('chat_members_user_chat_idx', 'user_id', 'chat_id'),)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
user_id: Mapped[UUID] = mapped_column(ForeignKey('users.id', ondelete='CASCADE'), primary_key=True)
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import DateTime, ForeignKey, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class MessageRow(Base):
__tablename__ = 'messages'
__table_args__ = (
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
UniqueConstraint(
'sender_id',
'chat_id',
'idempotency_key',
name='messages_sender_chat_idempotency_key',
),
)
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
sender_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
text: Mapped[str] = mapped_column(String(4000), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import DateTime, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from sqlalchemy.types import Uuid
from ..db.postgres import Base
class UserRow(Base):
__tablename__ = 'users'
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
username: Mapped[str] = mapped_column(String(32), unique=True, nullable=False)
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+6
View File
@@ -0,0 +1,6 @@
from .attachments import AttachmentRepository
from .chats import ChatRepository
from .messages import MessageRepository
from .users import UserRepository
__all__ = ['AttachmentRepository', 'ChatRepository', 'MessageRepository', 'UserRepository']
+93
View File
@@ -0,0 +1,93 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.models import Attachment
from ..models.attachment import AttachmentRow
from ..models.chat import ChatMemberRow, ChatRow
class AttachmentRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: AttachmentRow) -> Attachment:
return Attachment(
id=row.id,
chat_id=row.chat_id,
owner_id=row.owner_id,
filename=row.filename,
object_key=row.object_key,
content_type=row.content_type,
size_bytes=row.size_bytes,
sha256=row.sha256,
width=row.width,
height=row.height,
status=row.status,
error_code=row.error_code,
created_at=row.created_at,
updated_at=row.updated_at,
)
async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool:
async with self._sessions() as session:
found = await session.scalar(
select(ChatRow.id)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
)
return found is not None
async def create(
self,
*,
attachment_id: UUID,
chat_id: UUID,
owner_id: UUID,
filename: str,
object_key: str,
content_type: str,
size_bytes: int,
sha256: str,
width: int,
height: int,
now: datetime,
) -> Attachment:
row = AttachmentRow(
id=attachment_id,
chat_id=chat_id,
owner_id=owner_id,
filename=filename,
object_key=object_key,
content_type=content_type,
size_bytes=size_bytes,
sha256=sha256,
width=width,
height=height,
status='ready',
error_code=None,
created_at=now,
updated_at=now,
)
async with self._sessions() as session:
async with session.begin():
session.add(row)
await session.flush()
return self._to_domain(row)
async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None:
async with self._sessions() as session:
row = await session.scalar(
select(AttachmentRow)
.join(ChatMemberRow, ChatMemberRow.chat_id == AttachmentRow.chat_id)
.where(
AttachmentRow.id == attachment_id,
ChatMemberRow.user_id == user_id,
)
)
return self._to_domain(row) if row is not None else None
+69
View File
@@ -0,0 +1,69 @@
from __future__ import annotations
from datetime import datetime
from typing import Sequence
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.errors import Invalid
from ..domain.models import Chat, Keyset
from ..models.chat import ChatMemberRow, ChatRow
from ..models.user import UserRow
class ChatRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: ChatRow, member_ids: Sequence[UUID]) -> Chat:
return Chat(id=row.id, title=row.title, member_ids=tuple(member_ids), created_at=row.created_at)
@staticmethod
async def _members(session: AsyncSession, chat_id: UUID) -> tuple[UUID, ...]:
result = await session.scalars(
select(ChatMemberRow.user_id)
.where(ChatMemberRow.chat_id == chat_id)
.order_by(ChatMemberRow.user_id)
)
return tuple(result)
async def create(self, chat_id: UUID, title: str, member_ids: Sequence[UUID], created_at: datetime) -> Chat:
async with self._sessions() as session:
async with session.begin():
found = set(await session.scalars(select(UserRow.id).where(UserRow.id.in_(member_ids))))
if len(found) != len(member_ids):
raise Invalid(f'member_ids содержит неизвестных участников: {len(member_ids) - len(found)}')
row = ChatRow(id=chat_id, title=title, created_at=created_at)
session.add(row)
await session.flush()
session.add_all(ChatMemberRow(chat_id=chat_id, user_id=user_id) for user_id in member_ids)
await session.flush()
return self._to_domain(row, member_ids)
async def get_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None:
async with self._sessions() as session:
row = await session.scalar(
select(ChatRow)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
)
if row is None:
return None
return self._to_domain(row, await self._members(session, row.id))
async def page_for_member(self, user_id: UUID, after: Keyset | None, limit: int) -> list[Chat]:
async with self._sessions() as session:
query = (
select(ChatRow)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatMemberRow.user_id == user_id)
.order_by(ChatRow.created_at, ChatRow.id)
.limit(limit)
)
if after is not None:
query = query.where(tuple_(ChatRow.created_at, ChatRow.id) > tuple_(after.created_at, after.id))
rows = list((await session.scalars(query)).all())
return [self._to_domain(row, await self._members(session, row.id)) for row in rows]
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
from datetime import datetime
from typing import Sequence
from uuid import UUID
from sqlalchemy import select, tuple_
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.errors import Conflict, NotFound
from ..domain.models import Keyset, Message
from ..models.attachment import AttachmentRow, MessageAttachmentRow
from ..models.chat import ChatMemberRow, ChatRow
from ..models.message import MessageRow
class MessageRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: MessageRow, attachment_ids: Sequence[UUID]) -> Message:
return Message(
id=row.id,
chat_id=row.chat_id,
sender_id=row.sender_id,
text=row.text,
created_at=row.created_at,
attachment_ids=tuple(attachment_ids),
)
@staticmethod
async def _attachment_ids(session: AsyncSession, message_id: UUID) -> tuple[UUID, ...]:
result = await session.scalars(
select(MessageAttachmentRow.attachment_id)
.where(MessageAttachmentRow.message_id == message_id)
.order_by(MessageAttachmentRow.position)
)
return tuple(result)
@staticmethod
async def _accessible_chat_id(session: AsyncSession, chat_id: UUID, user_id: UUID) -> UUID | None:
return await session.scalar(
select(ChatRow.id)
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
)
async def create_for_member(
self,
message_id: UUID,
chat_id: UUID,
sender_id: UUID,
text: str,
attachment_ids: Sequence[UUID],
created_at: datetime,
idempotency_key: str,
) -> Message | None:
async with self._sessions() as session:
async with session.begin():
accessible_chat_id = await self._accessible_chat_id(session, chat_id, sender_id)
if accessible_chat_id is None:
return None
if attachment_ids:
found = {
attachment_id: status
for attachment_id, status in (
await session.execute(
select(AttachmentRow.id, AttachmentRow.status).where(
AttachmentRow.id.in_(attachment_ids),
AttachmentRow.chat_id == accessible_chat_id,
)
)
).all()
}
if set(found) != set(attachment_ids):
raise NotFound('вложение не найдено')
if any(status != 'ready' for status in found.values()):
raise Conflict('оригинал вложения недоступен')
statement = (
insert(MessageRow)
.values(
id=message_id,
chat_id=accessible_chat_id,
sender_id=sender_id,
text=text,
created_at=created_at,
idempotency_key=idempotency_key,
)
.on_conflict_do_nothing(
index_elements=[
MessageRow.sender_id,
MessageRow.chat_id,
MessageRow.idempotency_key,
]
)
.returning(MessageRow.id)
)
created_id = await session.scalar(statement)
if created_id is not None:
row = await session.get(MessageRow, created_id)
session.add_all(
MessageAttachmentRow(
message_id=created_id,
attachment_id=attachment_id,
position=position,
)
for position, attachment_id in enumerate(attachment_ids)
)
await session.flush()
result_attachment_ids = tuple(attachment_ids)
else:
row = await session.scalar(
select(MessageRow).where(
MessageRow.sender_id == sender_id,
MessageRow.chat_id == accessible_chat_id,
MessageRow.idempotency_key == idempotency_key,
)
)
if row is None:
raise RuntimeError('результат идемпотентной операции не найден')
result_attachment_ids = await self._attachment_ids(session, row.id)
if row.text != text or result_attachment_ids != tuple(attachment_ids):
raise Conflict('Idempotency-Key уже использован с другим сообщением')
return self._to_domain(row, result_attachment_ids)
async def page_for_member(
self, chat_id: UUID, user_id: UUID, after: Keyset | None, limit: int
) -> list[Message] | None:
async with self._sessions() as session:
accessible_chat_id = await self._accessible_chat_id(session, chat_id, user_id)
if accessible_chat_id is None:
return None
query = (
select(MessageRow)
.where(MessageRow.chat_id == accessible_chat_id)
.order_by(MessageRow.created_at, MessageRow.id)
.limit(limit)
)
if after is not None:
query = query.where(tuple_(MessageRow.created_at, MessageRow.id) > tuple_(after.created_at, after.id))
rows = list((await session.scalars(query)).all())
return [
self._to_domain(row, await self._attachment_ids(session, row.id))
for row in rows
]
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..domain.errors import Conflict
from ..domain.models import User
from ..models.user import UserRow
class UserRepository:
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
self._sessions = sessions
@staticmethod
def _to_domain(row: UserRow) -> User:
return User(id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at)
async def register(
self, user_id: UUID, username: str, display_name: str, password_hash: str, created_at: datetime
) -> User:
row = UserRow(
id=user_id,
username=username,
display_name=display_name,
password_hash=password_hash,
created_at=created_at,
)
try:
async with self._sessions() as session:
async with session.begin():
session.add(row)
await session.flush()
except IntegrityError as exc:
raise Conflict('username уже занят') from exc
return self._to_domain(row)
async def get_by_id(self, user_id: UUID) -> User | None:
async with self._sessions() as session:
row = await session.get(UserRow, user_id)
return self._to_domain(row) if row is not None else None
async def get_credentials(self, username: str) -> UserRow | None:
async with self._sessions() as session:
return await session.scalar(select(UserRow).where(UserRow.username == username))
@classmethod
def to_domain(cls, row: UserRow) -> User:
return cls._to_domain(row)
View File
+147
View File
@@ -0,0 +1,147 @@
from __future__ import annotations
import asyncio
import hashlib
import logging
from datetime import datetime, timezone
from tempfile import SpooledTemporaryFile
from typing import AsyncIterator, BinaryIO
import unicodedata
from uuid import UUID, uuid4
from PIL import Image, UnidentifiedImageError
from ..domain.errors import (
Conflict,
Invalid,
NotFound,
PayloadTooLarge,
Unavailable,
UnsupportedMediaType,
)
from ..domain.models import Attachment, User
from ..repositories import AttachmentRepository
from ..storage import ObjectStream, S3Storage
logger = logging.getLogger('app.attachments')
MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
def _validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255:
raise Invalid('filename должен содержать от 1 до 255 символов')
if any(
character in {'/', '\\'} or unicodedata.category(character) == 'Cc'
for character in filename
):
raise Invalid('filename содержит запрещённые символы')
def _inspect_image(source: BinaryIO, declared_type: str, max_pixels: int) -> tuple[str, int, int]:
try:
source.seek(0)
with Image.open(source) as image:
actual_type = MIME_BY_FORMAT.get(image.format or '')
if actual_type is None or actual_type != declared_type:
raise UnsupportedMediaType('MIME не соответствует содержимому PNG/JPEG')
width, height = image.size
if width <= 0 or height <= 0:
raise UnsupportedMediaType('изображение имеет неверные размеры')
if width * height > max_pixels:
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей')
image.verify()
except Image.DecompressionBombError as error:
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') from error
except (UnidentifiedImageError, OSError, SyntaxError) as error:
raise UnsupportedMediaType('повреждённое или неподдерживаемое изображение') from error
finally:
source.seek(0)
return actual_type, width, height
class AttachmentService:
def __init__(
self,
repository: AttachmentRepository,
storage: S3Storage,
max_upload_bytes: int,
max_image_pixels: int,
) -> None:
self._repository = repository
self._storage = storage
self._max_upload_bytes = max_upload_bytes
self._max_image_pixels = max_image_pixels
async def upload(
self,
actor: User,
chat_id: UUID,
filename: str,
declared_type: str,
chunks: AsyncIterator[bytes],
) -> Attachment:
_validate_filename(filename)
if declared_type not in MIME_BY_FORMAT.values():
raise UnsupportedMediaType('поддерживаются только image/png и image/jpeg')
if not await self._repository.chat_exists_for_member(chat_id, actor.id):
raise NotFound('чат не найден')
digest = hashlib.sha256()
size = 0
with SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b') as source:
async for chunk in chunks:
size += len(chunk)
if size > self._max_upload_bytes:
raise PayloadTooLarge('файл превышает лимит 10 MiB')
digest.update(chunk)
source.write(chunk)
if size == 0:
raise UnsupportedMediaType('пустой файл не является изображением')
actual_type, width, height = await asyncio.to_thread(
_inspect_image,
source,
declared_type,
self._max_image_pixels,
)
attachment_id = uuid4()
object_key = f'originals/{attachment_id}'
await self._storage.put(object_key, source, size, actual_type)
now = datetime.now(timezone.utc)
try:
return await self._repository.create(
attachment_id=attachment_id,
chat_id=chat_id,
owner_id=actor.id,
filename=filename,
object_key=object_key,
content_type=actual_type,
size_bytes=size,
sha256=digest.hexdigest(),
width=width,
height=height,
now=now,
)
except Exception:
try:
await self._storage.remove(object_key)
except Unavailable:
logger.warning('failed to compensate orphan object key=%s', object_key)
raise
async def get(self, actor: User, attachment_id: UUID) -> Attachment:
attachment = await self._repository.get_for_member(attachment_id, actor.id)
if attachment is None:
raise NotFound('вложение не найдено')
return attachment
async def content(self, actor: User, attachment_id: UUID) -> tuple[Attachment, ObjectStream]:
attachment = await self.get(actor, attachment_id)
if attachment.status != 'ready':
raise Conflict('оригинал вложения недоступен')
try:
stream = await self._storage.get(attachment.object_key)
except FileNotFoundError as error:
raise Unavailable('оригинал вложения отсутствует в хранилище') from error
return attachment, stream
+35
View File
@@ -0,0 +1,35 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Optional, Sequence
from uuid import UUID, uuid4
from ..domain.errors import Invalid, NotFound
from ..domain.models import Chat, Keyset, Page, User
from ..repositories import ChatRepository
class ChatService:
def __init__(self, repository: ChatRepository) -> None:
self._repository = repository
async def create(self, actor: User, title: str, member_ids: Sequence[UUID]) -> Chat:
if actor.id not in member_ids:
raise Invalid('вызывающий должен входить в member_ids')
return await self._repository.create(
chat_id=uuid4(),
title=title,
member_ids=member_ids,
created_at=datetime.now(timezone.utc),
)
async def get(self, actor: User, chat_id: UUID) -> Chat:
chat = await self._repository.get_for_member(chat_id, actor.id)
if chat is None:
raise NotFound('чат не найден')
return chat
async def page(self, actor: User, after: Optional[Keyset], limit: int) -> Page:
rows = await self._repository.page_for_member(actor.id, after, limit + 1)
items = tuple(rows[:limit])
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Optional, Sequence
from uuid import UUID, uuid4
from ..domain.errors import NotFound
from ..domain.models import Keyset, Message, Page, User
from ..repositories import MessageRepository
class MessageService:
def __init__(self, repository: MessageRepository) -> None:
self._repository = repository
async def send(
self,
actor: User,
chat_id: UUID,
text: str,
attachment_ids: Sequence[UUID],
idempotency_key: str,
) -> Message:
message = await self._repository.create_for_member(
message_id=uuid4(),
chat_id=chat_id,
sender_id=actor.id,
text=text,
attachment_ids=attachment_ids,
created_at=datetime.now(timezone.utc),
idempotency_key=idempotency_key,
)
if message is None:
raise NotFound('чат не найден')
return message
async def history(
self, actor: User, chat_id: UUID, after: Optional[Keyset], limit: int
) -> Page:
rows = await self._repository.page_for_member(chat_id, actor.id, after, limit + 1)
if rows is None:
raise NotFound('чат не найден')
items = tuple(rows[:limit])
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
+35
View File
@@ -0,0 +1,35 @@
from __future__ import annotations
from datetime import datetime, timezone
from uuid import UUID, uuid4
from ..auth.passwords import hash_password, password_matches
from ..domain.errors import Unauthorized
from ..domain.models import User
from ..repositories import UserRepository
class UserService:
def __init__(self, repository: UserRepository) -> None:
self._repository = repository
async def register(self, username: str, display_name: str, password: str) -> User:
return await self._repository.register(
user_id=uuid4(),
username=username,
display_name=display_name,
password_hash=hash_password(password),
created_at=datetime.now(timezone.utc),
)
async def identify(self, user_id: UUID) -> User:
user = await self._repository.get_by_id(user_id)
if user is None:
raise Unauthorized('требуется действующая сессия')
return user
async def authenticate(self, username: str, password: str) -> User | None:
row = await self._repository.get_credentials(username)
if row is None or not password_matches(row.password_hash, password):
return None
return self._repository.to_domain(row)
+3
View File
@@ -0,0 +1,3 @@
from .s3 import ObjectStream, S3Storage, StoredObject
__all__ = ['ObjectStream', 'S3Storage', 'StoredObject']
+132
View File
@@ -0,0 +1,132 @@
from __future__ import annotations
import asyncio
import certifi
from dataclasses import dataclass
from datetime import datetime
from typing import BinaryIO, Iterator
from urllib.parse import urlparse
from minio import Minio
from minio.error import InvalidResponseError, S3Error
from urllib3 import PoolManager, Retry, Timeout
from urllib3.exceptions import HTTPError
from ..domain.errors import Unavailable
def _unavailable(error: Exception) -> Unavailable:
return Unavailable('объектное хранилище недоступно')
@dataclass(slots=True)
class ObjectStream:
response: object
def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]:
try:
while chunk := self.response.read(size):
yield chunk
finally:
self.response.close()
self.response.release_conn()
@dataclass(frozen=True, slots=True)
class StoredObject:
key: str
last_modified: datetime
class S3Storage:
def __init__(
self,
endpoint: str,
access_key: str,
secret_key: str,
bucket: str,
region: str,
) -> None:
parsed = urlparse(endpoint)
if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}:
raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port')
self._client = Minio(
parsed.netloc,
access_key=access_key,
secret_key=secret_key,
secure=parsed.scheme == 'https',
region=region,
http_client=PoolManager(
timeout=Timeout(connect=2, read=5),
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
),
)
self.bucket = bucket
@staticmethod
def _translate(error: Exception) -> None:
if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}:
raise FileNotFoundError from error
if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)):
raise _unavailable(error) from error
raise error
async def ping(self) -> None:
try:
exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket)
except Exception as error:
self._translate(error)
if not exists:
raise Unavailable('bucket объектного хранилища не создан')
async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None:
source.seek(0)
try:
await asyncio.to_thread(
self._client.put_object,
self.bucket,
key,
source,
length,
content_type,
)
except Exception as error:
self._translate(error)
async def get(self, key: str) -> ObjectStream:
try:
response = await asyncio.to_thread(self._client.get_object, self.bucket, key)
except Exception as error:
self._translate(error)
return ObjectStream(response)
async def exists(self, key: str) -> bool:
try:
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
return True
except Exception as error:
try:
self._translate(error)
except FileNotFoundError:
return False
raise
async def remove(self, key: str) -> None:
try:
await asyncio.to_thread(self._client.remove_object, self.bucket, key)
except Exception as error:
self._translate(error)
async def list(self, prefix: str = '') -> tuple[StoredObject, ...]:
def collect() -> tuple[StoredObject, ...]:
return tuple(
StoredObject(item.object_name, item.last_modified)
for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True)
)
try:
return await asyncio.to_thread(collect)
except Exception as error:
self._translate(error)
+1
View File
@@ -0,0 +1 @@
"""Операторские CLI-инструменты приложения."""
+61
View File
@@ -0,0 +1,61 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage
async def cleanup(apply: bool, grace_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
sessions = create_session_factory(engine)
async with sessions() as session:
known = set(await session.scalars(select(AttachmentRow.object_key)))
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
removed = 0
for item in await storage.list('originals/'):
if item.key in known or item.last_modified > cutoff:
continue
action = 'DELETE' if apply else 'WOULD_DELETE'
print(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
if apply:
await storage.remove(item.key)
removed += 1
print(
f'summary mode={"apply" if apply else "dry-run"} '
f'known={len(known)} removed={removed} grace_seconds={grace_seconds}'
)
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(description='Safely find old S3 objects without PostgreSQL metadata')
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--grace-seconds', type=int, default=3600)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error('--grace-seconds must be at least 60')
raise SystemExit(asyncio.run(cleanup(args.apply, args.grace_seconds)))
if __name__ == '__main__':
main()
+250
View File
@@ -0,0 +1,250 @@
from __future__ import annotations
import argparse
import asyncio
import json
import time
import tracemalloc
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Callable, Sequence
from uuid import UUID
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..storage import S3Storage, StoredObject
MISSING_ERROR_CODE = 'original_missing'
@dataclass(frozen=True, slots=True)
class AttachmentSnapshot:
id: UUID
object_key: str
status: str
updated_at: datetime
@dataclass(frozen=True, slots=True)
class ReconcilePlan:
orphan: tuple[StoredObject, ...]
fresh_orphan: tuple[StoredObject, ...]
missing: tuple[AttachmentSnapshot, ...]
fresh_missing: tuple[AttachmentSnapshot, ...]
correct: int
def build_plan(
attachments: Sequence[AttachmentSnapshot],
objects: Sequence[StoredObject],
cutoff: datetime,
) -> ReconcilePlan:
"""Compare immutable PostgreSQL and S3 snapshots without changing either side."""
known_keys = {attachment.object_key for attachment in attachments}
object_keys = {item.key for item in objects}
unknown_objects = [item for item in objects if item.key not in known_keys]
orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff)
fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff)
ready_without_object = [
attachment
for attachment in attachments
if attachment.status == 'ready' and attachment.object_key not in object_keys
]
missing = tuple(
attachment for attachment in ready_without_object if attachment.updated_at <= cutoff
)
fresh_missing = tuple(
attachment for attachment in ready_without_object if attachment.updated_at > cutoff
)
correct = sum(attachment.object_key in object_keys for attachment in attachments)
return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct)
async def _load_attachments(
sessions: async_sessionmaker[AsyncSession],
) -> tuple[AttachmentSnapshot, ...]:
async with sessions() as session:
rows = (
await session.execute(
select(
AttachmentRow.id,
AttachmentRow.object_key,
AttachmentRow.status,
AttachmentRow.updated_at,
)
)
).all()
return tuple(AttachmentSnapshot(*row) for row in rows)
async def _metadata_key_exists(
sessions: async_sessionmaker[AsyncSession], object_key: str
) -> bool:
async with sessions() as session:
attachment_id = await session.scalar(
select(AttachmentRow.id).where(AttachmentRow.object_key == object_key).limit(1)
)
return attachment_id is not None
async def reconcile(
sessions: async_sessionmaker[AsyncSession],
storage: S3Storage,
*,
apply: bool,
grace_seconds: int,
now: datetime | None = None,
output: Callable[[str], None] = print,
) -> dict[str, int | float | str]:
started = time.perf_counter()
tracemalloc.start()
current_time = now or datetime.now(timezone.utc)
cutoff = current_time - timedelta(seconds=grace_seconds)
attachments = await _load_attachments(sessions)
objects = await storage.list('originals/')
plan = build_plan(attachments, objects, cutoff)
for item in plan.fresh_orphan:
output(
f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}'
)
for attachment in plan.fresh_missing:
output(
'SKIP_FRESH_MISSING '
f'attachment_id={attachment.id} key={attachment.object_key} '
f'updated={attachment.updated_at.isoformat()}'
)
removed = 0
race_protected = 0
for item in plan.orphan:
action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN'
if not apply:
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
continue
# PostgreSQL and S3 cannot share a transaction. Re-read the authoritative
# metadata immediately before a destructive action so a row created after
# the initial snapshot protects its object.
if await _metadata_key_exists(sessions, item.key):
output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}')
race_protected += 1
continue
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
await storage.remove(item.key)
removed += 1
marked_failed = 0
for attachment in plan.missing:
if not apply:
output(
f'WOULD_MARK_MISSING attachment_id={attachment.id} '
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
)
continue
# The object may have been restored after list_objects(). Do not turn
# healthy metadata into failed based only on the earlier snapshot.
if await storage.exists(attachment.object_key):
output(
f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} '
f'key={attachment.object_key}'
)
race_protected += 1
continue
async with sessions() as session:
async with session.begin():
result = await session.execute(
update(AttachmentRow)
.where(
AttachmentRow.id == attachment.id,
AttachmentRow.object_key == attachment.object_key,
AttachmentRow.status == 'ready',
AttachmentRow.updated_at == attachment.updated_at,
)
.values(
status='failed',
error_code=MISSING_ERROR_CODE,
updated_at=current_time,
)
)
changed = result.rowcount or 0
if changed:
output(
f'MARK_MISSING attachment_id={attachment.id} '
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
)
marked_failed += changed
else:
output(
f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} '
f'key={attachment.object_key}'
)
race_protected += 1
_, peak_bytes = tracemalloc.get_traced_memory()
tracemalloc.stop()
summary: dict[str, int | float | str] = {
'mode': 'apply' if apply else 'dry-run',
'metadata': len(attachments),
'objects': len(objects),
'correct': plan.correct,
'orphan': len(plan.orphan),
'missing': len(plan.missing),
'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing),
'removed': removed,
'marked_failed': marked_failed,
'race_protected': race_protected,
'grace_seconds': grace_seconds,
'elapsed_ms': round((time.perf_counter() - started) * 1000, 3),
'peak_memory_bytes': peak_bytes,
}
output(f'summary {json.dumps(summary, sort_keys=True)}')
return summary
async def run(apply: bool, grace_seconds: int) -> int:
settings = load_settings()
engine = create_engine(settings.database_url)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
await reconcile(
create_session_factory(engine),
storage,
apply=apply,
grace_seconds=grace_seconds,
)
return 0
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(
description='Reconcile attachment metadata in PostgreSQL with original objects in S3'
)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument('--dry-run', action='store_true')
mode.add_argument('--apply', action='store_true')
parser.add_argument('--grace-seconds', type=int, default=3600)
args = parser.parse_args()
if args.grace_seconds < 60:
parser.error('--grace-seconds must be at least 60')
raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds)))
if __name__ == '__main__':
main()
+136
View File
@@ -0,0 +1,136 @@
from __future__ import annotations
import argparse
import asyncio
from datetime import datetime, timedelta, timezone
from io import BytesIO
from uuid import NAMESPACE_URL, uuid5
from sqlalchemy import delete, insert
from ..config import load_settings
from ..db import create_engine, create_session_factory
from ..models.attachment import AttachmentRow
from ..models.chat import ChatMemberRow, ChatRow
from ..models.user import UserRow
from ..storage import S3Storage
USER_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:user')
CHAT_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:chat')
OBJECT_PREFIX = 'originals/reconciler-benchmark/'
WORKERS = 16
async def _put_objects(
storage: S3Storage,
keys: list[str],
) -> None:
for offset in range(0, len(keys), WORKERS):
batch = keys[offset : offset + WORKERS]
await asyncio.gather(
*(storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream') for key in batch)
)
async def prepare(size: int) -> None:
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
old = datetime.now(timezone.utc) - timedelta(hours=2)
rows = [
{
'id': uuid5(NAMESPACE_URL, f'reconciler-benchmark:attachment:{index}'),
'chat_id': CHAT_ID,
'owner_id': USER_ID,
'filename': f'benchmark-{index}.png',
'object_key': f'{OBJECT_PREFIX}{index}',
'content_type': 'image/png',
'size_bytes': 1,
'sha256': '0' * 64,
'width': 1,
'height': 1,
'status': 'ready',
'error_code': None,
'created_at': old,
'updated_at': old,
}
for index in range(size)
]
try:
async with sessions() as session:
async with session.begin():
session.add(UserRow(
id=USER_ID,
username=f'bench-{USER_ID.hex[:16]}',
display_name='Reconciler benchmark',
password_hash='not-used',
created_at=old,
))
session.add(ChatRow(id=CHAT_ID, title='Reconciler benchmark', created_at=old))
await session.flush()
session.add(ChatMemberRow(chat_id=CHAT_ID, user_id=USER_ID))
await session.flush()
for offset in range(0, size, 500):
await session.execute(insert(AttachmentRow), rows[offset : offset + 500])
await _put_objects(storage, [row['object_key'] for row in rows])
print(f'prepared metadata={size} objects={size}')
finally:
await engine.dispose()
async def cleanup() -> None:
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
try:
objects = await storage.list(OBJECT_PREFIX)
keys = [item.key for item in objects]
for offset in range(0, len(keys), WORKERS):
await asyncio.gather(
*(storage.remove(key) for key in keys[offset : offset + WORKERS])
)
async with sessions() as session:
async with session.begin():
await session.execute(delete(ChatRow).where(ChatRow.id == CHAT_ID))
await session.execute(delete(UserRow).where(UserRow.id == USER_ID))
print(f'cleaned objects={len(keys)}')
finally:
await engine.dispose()
def main() -> None:
parser = argparse.ArgumentParser(
description='Prepare and clean isolated PostgreSQL/S3 data for reconciler benchmarks'
)
subparsers = parser.add_subparsers(dest='command', required=True)
prepare_parser = subparsers.add_parser('prepare')
prepare_parser.add_argument('--size', type=int, required=True)
subparsers.add_parser('cleanup')
args = parser.parse_args()
if args.command == 'prepare':
if args.size < 1 or args.size > 10000:
parser.error('--size must be between 1 and 10000')
asyncio.run(prepare(args.size))
else:
asyncio.run(cleanup())
if __name__ == '__main__':
main()
+120
View File
@@ -0,0 +1,120 @@
services:
postgres:
image: postgres:17.4-alpine
environment:
POSTGRES_USER: messenger_test
POSTGRES_PASSWORD: messenger_test
POSTGRES_DB: messenger_test
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger_test -d messenger_test"]
interval: 2s
timeout: 2s
retries: 30
redis:
image: redis:7.4.2-alpine
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 2s
timeout: 2s
retries: 30
s3:
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
user: "0"
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: test-minio-admin
MINIO_ROOT_PASSWORD: test-minio-root-password
S3_BUCKET: messenger-attachments
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 2s
timeout: 2s
retries: 30
s3-bootstrap:
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
mc mb --ignore-existing local/"$$S3_BUCKET"
mc anonymous set none local/"$$S3_BUCKET"
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
mc admin policy create local messenger-api /tmp/s3-api-policy.json
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
environment:
MINIO_ROOT_USER: test-minio-admin
MINIO_ROOT_PASSWORD: test-minio-root-password
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
volumes:
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
depends_on:
s3:
condition: service_healthy
restart: "no"
migrate:
build: .
command: ["alembic", "upgrade", "head"]
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build: .
environment:
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
MAX_UPLOAD_BYTES: 10485760
MAX_IMAGE_PIXELS: 40000000
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
interval: 2s
timeout: 2s
retries: 30
tests:
build:
context: .
dockerfile: Dockerfile.test
environment:
BASE_URL: http://api:8080
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
REDIS_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: 3600
S3_ENDPOINT: http://s3:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID: messenger-api
S3_SECRET_ACCESS_KEY: test-minio-api-password
depends_on:
api:
condition: service_healthy
restart: "no"
+148
View File
@@ -0,0 +1,148 @@
services:
postgres:
image: postgres:17.4-alpine
cpus: "1.0"
mem_limit: 512m
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
interval: 5s
timeout: 3s
retries: 20
redis:
image: redis:7.4.2-alpine
cpus: "0.25"
mem_limit: 128m
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 20
s3:
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
user: "0"
cpus: "0.5"
mem_limit: 256m
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
volumes:
- s3_data:/data
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 5s
timeout: 3s
retries: 20
s3-bootstrap:
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
cpus: "0.25"
mem_limit: 128m
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
mc mb --ignore-existing local/"$$S3_BUCKET"
mc anonymous set none local/"$$S3_BUCKET"
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
mc admin policy create local messenger-api /tmp/s3-api-policy.json
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
volumes:
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
depends_on:
s3:
condition: service_healthy
restart: "no"
migrate:
build: .
cpus: "0.5"
mem_limit: 256m
command: ["alembic", "upgrade", "head"]
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build: .
cpus: "0.5"
mem_limit: 256m
environment:
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
LOG_LEVEL: ${LOG_LEVEL:-info}
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
expose:
- "8080"
deploy:
replicas: 2
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
s3-bootstrap:
condition: service_completed_successfully
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
interval: 2s
timeout: 2s
retries: 5
stop_grace_period: 10s
lb:
image: haproxy:3.2.26-alpine
cpus: "0.25"
mem_limit: 128m
ports:
- "${APP_PORT:-8080}:8080"
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"]
interval: 2s
timeout: 2s
retries: 5
stop_grace_period: 5s
volumes:
postgres_data:
redis_data:
s3_data:
+147
View File
@@ -0,0 +1,147 @@
# Результаты проверки lab4
Дата последней перепроверки: 2026-11-05. Стенд: Apple M4 Pro, 12 CPU cores, 24 GB RAM, Docker Desktop.
## Версии runtime
```text
Python 3.13.15
PostgreSQL 17.4
Redis 7.4.2
HAProxy 3.2.26
MinIO RELEASE.2026-10-23T20-06-51Z (commit 76c06ff1d2cc844050fdeb1d54bf8fa70db72278)
MinIO Client DEVELOPMENT.GOGET
```
MinIO и MinIO Client взяты из Chainguard Images и закреплены в Compose по multi-arch OCI index digest.
## Контракт и smoke
```text
$ make check
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ make validate PYTHON=.venv/bin/python
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
$ make test
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
```
## Негативные сценарии вложений
```text
$ make test-attachments PYTHON=.venv/bin/python
............ [100%]
12 passed in 1.86s
```
Проверены: превышение 10 MiB, повреждённое изображение, подмена MIME, превышение 40 млн пикселей до декодирования, опасные filename, доступ постороннего, attachment другого чата и семантика идемпотентности `attachment_ids`.
```text
$ docker compose -f compose.tests.yaml run --rm --no-deps tests python -m pytest -q tests/test_attachment_compensation.py
. [100%]
1 passed
```
Unit-тест внедряет ошибку сохранения метаданных после успешного S3 PUT и проверяет компенсационный DELETE того же object key.
Полный запуск в чистом изолированном Compose-проекте, включая миграции и bootstrap MinIO:
```text
$ docker compose -p lab4-chainguard-tests -f compose.tests.yaml run --rm tests python -m pytest -q
............. [100%]
13 passed in 0.41s
```
После добавления reconciler трека 2 итоговый запуск в отдельном Compose-проекте:
```text
$ docker compose -p lab4-review3 -f compose.tests.yaml run --build --rm tests
................. [100%]
17 passed in 0.60s
```
## Приватность и bootstrap
```text
$ scripts/check-private-bucket.sh
PASS: private bucket rejected anonymous request with HTTP 403
```
MinIO root credentials получает только server/bootstrap. API использует отдельного пользователя с bucket-scoped policy. Host-порты S3 API и консоли не опубликованы. Bootstrap повторно завершался с code 0 при восстановлении API после failover и при restart стенда.
## Persistence и отказы
```text
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
```
Hook перезапустил API, PostgreSQL, Redis и MinIO с сохранением named volumes.
```text
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 1, "first_recovery_seconds": 2.333}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
```
```text
$ make test-attachment-failover ACTION_SCRIPT=scripts/stop-one.sh
PASS: upload_instance=c18e79906d77 download_instance=662967386db9 recovery_seconds=3.912 failures=1 sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75
```
В последнем сценарии остановлена именно реплика из `X-Instance-Id` ответа upload. Download выполнен через другую реплику и совпал побайтно.
```text
$ scripts/check-s3-outage.sh
PASS: API readiness returned structured 503 while S3 was unavailable
```
Скрипт автоматически восстановил MinIO и дождался готовности стенда.
## Cleanup
```text
$ python -m app.tools.cleanup_orphans --dry-run --grace-seconds 60
summary mode=dry-run known=164 removed=0 grace_seconds=60
```
Dry-run не менял данные. Применение требует явного `--apply`; значения grace period меньше 60 секунд запрещены.
## Reconciler PostgreSQL/S3 — трек 2
`tests/test_reconciler_integration.py` внедряет ошибки на обеих сторонах настоящего
тестового стенда: старую metadata без объекта (`missing`) и объект без metadata
(`orphan`). Контрольное вложение связано с сообщением.
```text
$ docker compose -p lab4-review3 -f compose.tests.yaml run --rm tests \
python -m pytest -q tests/test_reconciler.py tests/test_reconciler_integration.py
... [100%]
3 passed in 0.30s
```
Проверено: dry-run ничего не меняет; apply удаляет один подтверждённый orphan и переводит
один подтверждённый missing в `failed/original_missing`. Missing-объект, восстановленный
после listing, missing-строка с изменившимся `updated_at` и orphan, для которого metadata
появилась после снимка, получают `SKIP_*` и учитываются в `race_protected`. Отдельный
свежий orphan переживает настоящий apply.
Повторный apply выполняет 0 повторных исправлений. Объект используемого вложения и связь
с сообщением сохранены.
Воспроизводимый end-to-end benchmark самостоятельно подготовил реальные данные в
PostgreSQL и MinIO, а после измерения удалил их:
```text
$ SIZES="100 1000 10000" RUNS=3 ./scripts/benchmark-reconciler.sh
size=100: elapsed_ms=78.847,80.712,82.781; peak median=2620657
size=1000: elapsed_ms=232.254,227.467,226.264; peak median=4205016
size=10000: elapsed_ms=1701.005,1792.955,1733.545; peak median=8075853
```
Медианы времени: 80,712; 227,467; 1733,545 мс. Скрипт готовит каждый размер один раз,
делает три dry-run, затем удаляет benchmark-фикстуру. Измерение выполнено в отдельном
Compose-проекте; фиксированные три пары, созданные интеграционными HTTP-проверками,
присутствовали во всех точках одинаково и не влияют на вывод о масштабе.
+20
View File
@@ -0,0 +1,20 @@
{
"date": "2026-11-04",
"profile": {
"concurrency": 10,
"uploads": 30,
"file_size_bytes": 111817,
"api_memory_limit_mib_each": 256
},
"result": {
"successful_uploads": 30,
"observed_api_instances": 2,
"duration_seconds": 0.382,
"max_latency_seconds": 0.3,
"peak_memory_mib": {
"api_replica_1": 68.5,
"api_replica_2": 68.33
}
},
"limitations": "Closed-loop profile with a small PNG; docker stats is sampled and may miss a shorter transient peak."
}
+31
View File
@@ -0,0 +1,31 @@
global
log stdout format raw local0
defaults
log global
mode http
option httplog
timeout connect 2s
timeout client 30s
timeout server 30s
timeout check 2s
resolvers docker
nameserver docker_dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
frontend public
bind :8080
default_backend api_backend
backend api_backend
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
server-template api 2 api:8080
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::BUCKET_NAME"]
},
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::BUCKET_NAME/*"]
}
]
}
+1
View File
@@ -1,2 +1,3 @@
# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
openapi-spec-validator==0.7.2
pytest==8.3.5
+9
View File
@@ -0,0 +1,9 @@
fastapi==0.115.12
uvicorn[standard]==0.34.2
SQLAlchemy[asyncio]==2.0.40
asyncpg==0.30.0
alembic==1.15.2
redis==5.2.1
argon2-cffi==25.1.0
minio==7.2.20
Pillow==13.0.0
+26
View File
@@ -0,0 +1,26 @@
#!/bin/sh
set -eu
sizes="${SIZES:-${SIZE:-100 1000 10000}}"
runs="${RUNS:-3}"
case "$runs" in
''|*[!0-9]*) printf '%s\n' 'RUNS must be a positive integer' >&2; exit 2 ;;
0) printf '%s\n' 'RUNS must be a positive integer' >&2; exit 2 ;;
esac
cleanup() {
docker compose exec -T api python -m app.tools.seed_reconciler_benchmark cleanup
}
trap cleanup EXIT HUP INT TERM
for size in $sizes; do
cleanup
docker compose exec -T api python -m app.tools.seed_reconciler_benchmark prepare --size "$size"
run=1
while [ "$run" -le "$runs" ]; do
printf 'benchmark size=%s run=%s/%s\n' "$size" "$run" "$runs"
docker compose exec -T api python -m app.tools.reconcile_attachments --dry-run
run=$((run + 1))
done
done
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
status=$(docker compose --env-file .env exec -T api python -c '
import os
import urllib.error
import urllib.request
url = "http://s3:9000/" + os.environ["S3_BUCKET"] + "/"
try:
response = urllib.request.urlopen(url, timeout=5)
except urllib.error.HTTPError as error:
print(error.code)
else:
print(response.status)
')
case "$status" in
401|403) printf '%s\n' "PASS: private bucket rejected anonymous request with HTTP $status" ;;
*) printf '%s\n' "FAIL: anonymous bucket request returned HTTP $status" >&2; exit 1 ;;
esac
+29
View File
@@ -0,0 +1,29 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
restore() {
docker compose --env-file .env up -d --wait --wait-timeout 60 s3 api lb >/dev/null
}
trap restore EXIT HUP INT TERM
docker compose --env-file .env stop s3 >/dev/null
docker compose --env-file .env exec -T api python -c '
import json
import urllib.error
import urllib.request
try:
urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=25)
except urllib.error.HTTPError as error:
body = json.loads(error.read())
assert error.code == 503, error.code
assert body["error"]["code"] == "unavailable", body
else:
raise SystemExit("readiness unexpectedly succeeded while S3 was stopped")
'
printf '%s\n' 'PASS: API readiness returned structured 503 while S3 was unavailable'
+109
View File
@@ -0,0 +1,109 @@
#!/usr/bin/env python3
"""Run concurrent uploads while sampling API container memory with docker stats."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import re
import subprocess
import sys
import threading
ROOT = Path(__file__).resolve().parents[1]
UNITS = {
'B': 1,
'kB': 1000,
'KB': 1000,
'KiB': 1024,
'MB': 1000**2,
'MiB': 1024**2,
'GB': 1000**3,
'GiB': 1024**3,
}
def bytes_value(value: str) -> int:
match = re.fullmatch(r'([0-9.]+)([A-Za-z]+)', value.strip())
if match is None or match.group(2) not in UNITS:
raise ValueError(f'unsupported docker memory value: {value}')
return round(float(match.group(1)) * UNITS[match.group(2)])
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--uploads', type=int, default=30)
args = parser.parse_args()
container_ids = subprocess.check_output(
['docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api'],
cwd=ROOT,
text=True,
).split()
if len(container_ids) < 2:
raise SystemExit('expected at least two API containers')
peaks: dict[str, int] = {}
stop = threading.Event()
ready = threading.Event()
def collect() -> None:
while not stop.is_set():
output = subprocess.check_output(
['docker', 'stats', '--no-stream', '--format', '{{json .}}', *container_ids],
cwd=ROOT,
text=True,
)
for line in output.splitlines():
sample = json.loads(line)
name, usage = sample['Name'], sample['MemUsage']
current = bytes_value(usage.split('/', 1)[0])
peaks[name] = max(peaks.get(name, 0), current)
ready.set()
reader = threading.Thread(target=collect, daemon=True)
reader.start()
if not ready.wait(timeout=15):
raise SystemExit('docker stats did not produce an initial sample')
try:
load = subprocess.run(
[
sys.executable,
'tests/upload_load.py',
'--base-url',
args.base_url,
'--concurrency',
str(args.concurrency),
'--uploads',
str(args.uploads),
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
finally:
stop.set()
reader.join(timeout=15)
if not peaks:
raise SystemExit('docker stats produced no samples; increase --uploads')
print(load.stdout.strip())
print(
json.dumps(
{
'peak_memory_bytes': peaks,
'peak_memory_mib': {
name: round(value / 1024 / 1024, 2) for name, value in peaks.items()
},
},
ensure_ascii=False,
)
)
if __name__ == '__main__':
main()
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
# Останавливаем публичный маршрут до перезапуска общих хранилищ.
docker compose --env-file .env stop lb api >/dev/null
docker compose --env-file .env restart postgres redis s3 >/dev/null
docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 >/dev/null
docker compose --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
project_dir=$(dirname -- "$script_dir")
cd "$project_dir"
target_container=''
for container_id in $(docker compose --env-file .env ps -q api); do
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
target_container=$container_id
break
fi
done
if [ -z "$target_container" ]; then
printf '%s\n' 'API instance from TARGET_INSTANCE_ID was not found' >&2
exit 1
fi
docker stop --time 5 "$target_container" >/dev/null
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""Stop the API replica that accepted upload and download through another replica."""
import hashlib
import os
from pathlib import Path
import subprocess
import time
from client import Client, ROOT, arguments, check, run
def main():
def configure(parser):
parser.add_argument('--action-script', required=True)
args = arguments('Attachment API-replica failover', configure)
script = Path(args.action_script).expanduser().resolve()
check(script.is_file() and os.access(script, os.X_OK), 'action script must be executable')
client = Client(args.base_url, args.ca_file, timeout=3)
alice, bob, _eve, chat = client.bootstrap()
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
uploaded = client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/attachments?filename=failover.png',
201,
actor=alice,
raw=content,
headers={'Content-Type': 'image/png'},
schema='Attachment',
)
target = uploaded['headers']['X-Instance-Id']
env = dict(os.environ)
env['TARGET_INSTANCE_ID'] = target
started = time.monotonic()
subprocess.run([str(script)], env=env, check=True)
deadline = time.monotonic() + 15
failures = 0
while True:
try:
downloaded = client.call(
'GET',
f'/api/v1/attachments/{uploaded["json"]["id"]}/content',
actor=bob,
)
if downloaded['headers']['X-Instance-Id'] != target:
break
except Exception:
failures += 1
check(time.monotonic() < deadline, 'download did not recover through another replica')
time.sleep(0.2)
check(downloaded['bytes'] == content, 'downloaded file differs after replica failure')
print(
'PASS: upload_instance=%s download_instance=%s recovery_seconds=%.3f failures=%d sha256=%s'
% (
target,
downloaded['headers']['X-Instance-Id'],
time.monotonic() - started,
failures,
hashlib.sha256(content).hexdigest(),
)
)
if __name__ == '__main__':
run(main)
+98
View File
@@ -0,0 +1,98 @@
"""Проверка компенсации объекта при ошибке commit метаданных."""
import asyncio
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
import pytest
from app.domain.errors import Conflict
from app.domain.models import Attachment
from app.domain.models import User
from app.services.attachments import AttachmentService
class FailingRepository:
async def chat_exists_for_member(self, chat_id, user_id):
return True
async def create(self, **kwargs):
raise RuntimeError('injected database commit failure')
class RecordingStorage:
def __init__(self):
self.put_key = None
self.removed_key = None
async def put(self, key, source, length, content_type):
self.put_key = key
async def remove(self, key):
self.removed_key = key
class FailedAttachmentRepository:
def __init__(self, attachment):
self.attachment = attachment
async def get_for_member(self, attachment_id, user_id):
return self.attachment
class StorageThatMustNotBeRead:
async def get(self, key):
raise AssertionError('failed attachment must not read S3')
def test_uploaded_object_is_removed_when_metadata_commit_fails():
"""Удаляет объект после ошибки сохранения метаданных."""
storage = RecordingStorage()
service = AttachmentService(FailingRepository(), storage, 10 * 1024 * 1024, 40_000_000)
actor = User(uuid4(), 'tester', 'Tester', datetime.now(timezone.utc))
content = (Path(__file__).parent / 'fixtures/sample.png').read_bytes()
async def chunks():
yield content
with pytest.raises(RuntimeError, match='injected database commit failure'):
asyncio.run(
service.upload(
actor,
uuid4(),
'sample.png',
'image/png',
chunks(),
)
)
assert storage.put_key is not None
assert storage.removed_key == storage.put_key
def test_failed_attachment_content_returns_conflict_without_s3_read():
"""Возвращает конфликт для ошибочного вложения без чтения S3."""
now = datetime.now(timezone.utc)
attachment = Attachment(
id=uuid4(),
chat_id=uuid4(),
owner_id=uuid4(),
filename='missing.png',
object_key=f'originals/{uuid4()}',
content_type='image/png',
size_bytes=1,
sha256='0' * 64,
width=1,
height=1,
status='failed',
error_code='original_missing',
created_at=now,
updated_at=now,
)
service = AttachmentService(
FailedAttachmentRepository(attachment), StorageThatMustNotBeRead(), 1, 1
)
actor = User(uuid4(), 'tester', 'Tester', now)
with pytest.raises(Conflict, match='недоступен'):
asyncio.run(service.content(actor, attachment.id))
+153
View File
@@ -0,0 +1,153 @@
"""Негативные интеграционные проверки вложений для уровня 4."""
import struct
from urllib.parse import quote
import uuid
import zlib
import pytest
from tests.client import Client, ROOT
@pytest.fixture(scope='module')
def scenario():
client = Client()
alice, bob, eve, chat = client.bootstrap()
return client, alice, bob, eve, chat
def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=201):
return client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/attachments?filename={quote(filename, safe="")}',
expected,
actor=actor,
raw=content,
headers={'Content-Type': content_type},
schema='Attachment' if expected == 201 else None,
)
def oversized_dimensions_png(content: bytes) -> bytes:
data = bytearray(content)
data[16:20] = struct.pack('>I', 10_000)
data[20:24] = struct.pack('>I', 5_000)
data[29:33] = struct.pack('>I', zlib.crc32(data[12:29]) & 0xFFFFFFFF)
return bytes(data)
def test_file_larger_than_10_mib_is_rejected(scenario):
"""Отклоняет файл размером больше 10 MiB."""
client, alice, _bob, _eve, chat = scenario
upload(client, alice, chat, b'x' * (10 * 1024 * 1024 + 1), expected=413)
def test_corrupted_image_is_rejected(scenario):
"""Отклоняет повреждённое изображение."""
client, alice, _bob, _eve, chat = scenario
upload(client, alice, chat, b'not-a-png', expected=415)
def test_mime_mismatch_is_rejected(scenario):
"""Отклоняет MIME, не совпадающий с содержимым."""
client, alice, _bob, _eve, chat = scenario
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
upload(client, alice, chat, content, content_type='image/jpeg', expected=415)
def test_pixel_limit_is_checked_before_decode(scenario):
"""Проверяет лимит пикселей до полной декомпрессии."""
client, alice, _bob, _eve, chat = scenario
content = oversized_dimensions_png((ROOT / 'tests/fixtures/sample.png').read_bytes())
upload(client, alice, chat, content, expected=413)
@pytest.mark.parametrize(
'filename',
['../escape.png', 'folder\\escape.png', 'bad\x00name.png', 'bad\x85name.png'],
)
def test_dangerous_filename_is_rejected(scenario, filename):
"""Отклоняет опасное отображаемое имя файла."""
client, alice, _bob, _eve, chat = scenario
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
upload(client, alice, chat, content, filename=filename, expected=400)
def test_outsider_cannot_read_attachment(scenario):
"""Скрывает метаданные и содержимое от постороннего пользователя."""
client, alice, _bob, eve, chat = scenario
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
attachment = upload(client, alice, chat, content)['json']
client.call('GET', f'/api/v1/attachments/{attachment["id"]}', 404, actor=eve)
client.call('GET', f'/api/v1/attachments/{attachment["id"]}/content', 404, actor=eve)
def test_attachment_from_another_chat_cannot_be_linked(scenario):
"""Запрещает прикреплять вложение другого чата."""
client, alice, bob, _eve, chat = scenario
other_chat = client.call(
'POST',
'/api/v1/chats',
201,
actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]},
schema='Chat',
)['json']
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
attachment = upload(client, alice, other_chat, content)['json']
client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/messages',
404,
actor=alice,
body={'text': 'чужое вложение', 'attachment_ids': [attachment['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex},
)
def test_idempotency_distinguishes_attachment_list(scenario):
"""Включает список вложений в сравнение идемпотентного запроса."""
client, alice, _bob, _eve, chat = scenario
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
attachment = upload(client, alice, chat, content)['json']
key = uuid.uuid4().hex
first = client.message(
alice,
chat,
'идемпотентное вложение',
key=key,
attachments=[attachment['id']],
)['json']
replay = client.message(
alice,
chat,
'идемпотентное вложение',
key=key,
attachments=[attachment['id']],
)['json']
assert replay == first
client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/messages',
409,
actor=alice,
body={'text': 'идемпотентное вложение', 'attachment_ids': []},
headers={'Idempotency-Key': key},
)
def test_missing_attachment_ids_equals_empty_list_for_idempotency(scenario):
"""Считает отсутствие attachment_ids эквивалентным пустому списку."""
client, alice, _bob, _eve, chat = scenario
key = uuid.uuid4().hex
first = client.message(alice, chat, 'обычный текст', key=key)['json']
replay = client.call(
'POST',
f'/api/v1/chats/{chat["id"]}/messages',
201,
actor=alice,
body={'text': 'обычный текст', 'attachment_ids': []},
headers={'Idempotency-Key': key},
schema='Message',
)['json']
assert replay == first
+52
View File
@@ -0,0 +1,52 @@
from datetime import datetime, timedelta, timezone
from uuid import uuid4
from app.storage import StoredObject
from app.tools.reconcile_attachments import AttachmentSnapshot, build_plan
def test_plan_classifies_both_sides_and_protects_fresh_items():
"""Классифицирует orphan/missing и защищает свежие расхождения."""
now = datetime.now(timezone.utc)
cutoff = now - timedelta(minutes=1)
correct_key = f'originals/{uuid4()}'
missing = AttachmentSnapshot(uuid4(), f'originals/{uuid4()}', 'ready', cutoff)
fresh_missing = AttachmentSnapshot(uuid4(), f'originals/{uuid4()}', 'ready', now)
already_failed = AttachmentSnapshot(uuid4(), f'originals/{uuid4()}', 'failed', cutoff)
attachments = (
AttachmentSnapshot(uuid4(), correct_key, 'ready', cutoff),
missing,
fresh_missing,
already_failed,
)
old_orphan = StoredObject(f'originals/{uuid4()}', cutoff)
fresh_orphan = StoredObject(f'originals/{uuid4()}', now)
plan = build_plan(
attachments,
(StoredObject(correct_key, cutoff), old_orphan, fresh_orphan),
cutoff,
)
assert plan.correct == 1
assert plan.orphan == (old_orphan,)
assert plan.fresh_orphan == (fresh_orphan,)
assert plan.missing == (missing,)
assert plan.fresh_missing == (fresh_missing,)
def test_failed_metadata_still_protects_its_object_from_orphan_cleanup():
"""Проверяет защиту объекта существующей failed-metadata."""
now = datetime.now(timezone.utc)
key = f'originals/{uuid4()}'
failed = AttachmentSnapshot(uuid4(), key, 'failed', now - timedelta(hours=2))
plan = build_plan(
(failed,),
(StoredObject(key, now - timedelta(hours=2)),),
now - timedelta(hours=1),
)
assert plan.correct == 1
assert plan.orphan == ()
assert plan.missing == ()
+273
View File
@@ -0,0 +1,273 @@
"""Real PostgreSQL + S3 fault injection for grade 5, track 2."""
from __future__ import annotations
import asyncio
import os
from datetime import datetime, timedelta, timezone
from io import BytesIO
from uuid import uuid4
import pytest
from sqlalchemy import select, update
from app.config import load_settings
from app.db import create_engine, create_session_factory
from app.models.attachment import AttachmentRow, MessageAttachmentRow
from app.models.chat import ChatMemberRow, ChatRow
from app.models.message import MessageRow
from app.models.user import UserRow
from app.storage import S3Storage
from app.tools.reconcile_attachments import MISSING_ERROR_CODE, reconcile
pytestmark = pytest.mark.skipif(
not os.getenv('DATABASE_URL'), reason='requires the compose PostgreSQL and S3 services'
)
class AfterListStorage:
def __init__(self, storage, after_list):
self._storage = storage
self._after_list = after_list
async def list(self, prefix=''):
snapshot = await self._storage.list(prefix)
if self._after_list is not None:
after_list, self._after_list = self._after_list, None
await after_list()
return snapshot
def __getattr__(self, name):
return getattr(self._storage, name)
def test_dry_run_apply_and_repeat_are_safe():
"""Проверяет пробный, изменяющий и повторный запуски."""
asyncio.run(_check_dry_run_apply_and_repeat_are_safe())
async def _check_dry_run_apply_and_repeat_are_safe():
settings = load_settings()
engine = create_engine(settings.database_url)
sessions = create_session_factory(engine)
storage = S3Storage(
settings.s3_endpoint,
settings.s3_access_key_id,
settings.s3_secret_access_key,
settings.s3_bucket,
settings.s3_region,
)
now = datetime.now(timezone.utc)
old = now - timedelta(hours=2)
simulated_after_grace = now + timedelta(seconds=61)
user_id, chat_id, message_id = uuid4(), uuid4(), uuid4()
correct_id, missing_id, restored_id = uuid4(), uuid4(), uuid4()
changed_id, appeared_id = uuid4(), uuid4()
correct_key = f'originals/reconcile-correct-{uuid4()}'
orphan_key = f'originals/reconcile-orphan-{uuid4()}'
restored_key = f'originals/reconcile-restored-{uuid4()}'
changed_key = f'originals/reconcile-changed-{uuid4()}'
appeared_key = f'originals/reconcile-appeared-{uuid4()}'
fresh_key = f'originals/reconcile-fresh-{uuid4()}'
try:
async with sessions() as session:
async with session.begin():
session.add(UserRow(
id=user_id,
username=f'reconcile-{user_id.hex[:16]}',
display_name='Reconciler test',
password_hash='not-used',
created_at=old,
))
session.add(ChatRow(id=chat_id, title='Reconciler test', created_at=old))
await session.flush()
session.add(ChatMemberRow(chat_id=chat_id, user_id=user_id))
for attachment_id, key in (
(correct_id, correct_key),
(missing_id, f'originals/reconcile-missing-{uuid4()}'),
(restored_id, restored_key),
(changed_id, changed_key),
):
session.add(AttachmentRow(
id=attachment_id,
chat_id=chat_id,
owner_id=user_id,
filename='test.png',
object_key=key,
content_type='image/png',
size_bytes=1,
sha256='0' * 64,
width=1,
height=1,
status='ready',
error_code=None,
created_at=old,
updated_at=old,
))
await session.flush()
session.add(MessageRow(
id=message_id,
chat_id=chat_id,
sender_id=user_id,
text='used attachment must survive',
created_at=old,
idempotency_key=f'reconcile-{message_id}',
))
await session.flush()
session.add(MessageAttachmentRow(
message_id=message_id,
attachment_id=correct_id,
position=0,
))
for key in (correct_key, orphan_key):
await storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream')
dry_run = await reconcile(
sessions,
storage,
apply=False,
grace_seconds=60,
now=simulated_after_grace,
output=lambda _line: None,
)
assert dry_run['orphan'] == 1
assert dry_run['missing'] == 3
assert dry_run['removed'] == 0
assert dry_run['marked_failed'] == 0
assert {item.key for item in await storage.list('originals/')} >= {
correct_key,
orphan_key,
}
async def restore_missing_after_snapshot():
await storage.put(restored_key, BytesIO(b'x'), 1, 'application/octet-stream')
async with sessions() as session:
async with session.begin():
await session.execute(
update(AttachmentRow)
.where(AttachmentRow.id == changed_id)
.values(updated_at=now)
)
applied = await reconcile(
sessions,
AfterListStorage(storage, restore_missing_after_snapshot),
apply=True,
grace_seconds=60,
now=simulated_after_grace,
output=lambda _line: None,
)
assert applied['removed'] == 1
assert applied['marked_failed'] == 1
assert applied['race_protected'] == 2
# Make the concurrently changed metadata healthy before checking that
# the next complete run has no work left.
await storage.put(changed_key, BytesIO(b'x'), 1, 'application/octet-stream')
repeated = await reconcile(
sessions,
storage,
apply=True,
grace_seconds=60,
now=simulated_after_grace,
output=lambda _line: None,
)
assert repeated['orphan'] == 0
assert repeated['missing'] == 0
assert repeated['removed'] == 0
assert repeated['marked_failed'] == 0
object_keys = {item.key for item in await storage.list('originals/')}
assert correct_key in object_keys
assert restored_key in object_keys
assert changed_key in object_keys
assert orphan_key not in object_keys
async with sessions() as session:
missing_row = await session.get(AttachmentRow, missing_id)
assert missing_row is not None
assert missing_row.status == 'failed'
assert missing_row.error_code == MISSING_ERROR_CODE
assert await session.scalar(
select(MessageAttachmentRow.attachment_id).where(
MessageAttachmentRow.message_id == message_id
)
) == correct_id
restored_row = await session.get(AttachmentRow, restored_id)
assert restored_row is not None
assert restored_row.status == 'ready'
changed_row = await session.get(AttachmentRow, changed_id)
assert changed_row is not None
assert changed_row.status == 'ready'
# An object younger than the grace period survives a real apply run.
await storage.put(fresh_key, BytesIO(b'x'), 1, 'application/octet-stream')
fresh_apply = await reconcile(
sessions,
storage,
apply=True,
grace_seconds=60,
now=datetime.now(timezone.utc),
output=lambda _line: None,
)
assert fresh_apply['fresh_protected'] == 1
assert fresh_apply['removed'] == 0
assert await storage.exists(fresh_key)
await storage.remove(fresh_key)
# Metadata created after the initial snapshots also protects an old
# object from the destructive orphan action.
await storage.put(appeared_key, BytesIO(b'x'), 1, 'application/octet-stream')
async def create_metadata_after_snapshot():
async with sessions() as session:
async with session.begin():
session.add(AttachmentRow(
id=appeared_id,
chat_id=chat_id,
owner_id=user_id,
filename='appeared.png',
object_key=appeared_key,
content_type='image/png',
size_bytes=1,
sha256='0' * 64,
width=1,
height=1,
status='ready',
error_code=None,
created_at=old,
updated_at=old,
))
appeared_apply = await reconcile(
sessions,
AfterListStorage(storage, create_metadata_after_snapshot),
apply=True,
grace_seconds=60,
now=datetime.now(timezone.utc) + timedelta(seconds=61),
output=lambda _line: None,
)
assert appeared_apply['orphan'] == 1
assert appeared_apply['removed'] == 0
assert appeared_apply['race_protected'] == 1
assert await storage.exists(appeared_key)
finally:
await storage.remove(correct_key)
await storage.remove(orphan_key)
await storage.remove(restored_key)
await storage.remove(changed_key)
await storage.remove(appeared_key)
await storage.remove(fresh_key)
async with sessions() as session:
async with session.begin():
chat = await session.get(ChatRow, chat_id)
if chat is not None:
await session.delete(chat)
await session.flush()
user = await session.get(UserRow, user_id)
if user is not None:
await session.delete(user)
await engine.dispose()
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env python3
"""Простой профиль параллельных загрузок для измерения RAM контейнеров API."""
from concurrent.futures import ThreadPoolExecutor
import json
import time
import uuid
from client import Client, ROOT, arguments, run
def main():
def configure(parser):
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--uploads', type=int, default=30)
args = arguments(
'Parallel attachment uploads',
configure,
)
if args.concurrency < 10:
raise ValueError('concurrency must be at least 10 for lab4 evidence')
if args.uploads < args.concurrency:
raise ValueError('uploads must be greater than or equal to concurrency')
client = Client(args.base_url, args.ca_file, timeout=30)
alice, _bob, _eve, chat = client.bootstrap()
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
def upload(index):
local = Client(args.base_url, args.ca_file, timeout=30)
started = time.monotonic()
response = local.call(
'POST',
f'/api/v1/chats/{chat["id"]}/attachments?filename=load-{index}-{uuid.uuid4().hex}.png',
201,
actor=alice,
raw=content,
headers={'Content-Type': 'image/png'},
schema='Attachment',
)
return response['headers']['X-Instance-Id'], time.monotonic() - started
started = time.monotonic()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
results = list(pool.map(upload, range(args.uploads)))
duration = time.monotonic() - started
latencies = sorted(item[1] for item in results)
print(
json.dumps(
{
'concurrency': args.concurrency,
'file_size_bytes': len(content),
'successful_uploads': len(results),
'instances': len({item[0] for item in results}),
'duration_seconds': round(duration, 3),
'max_latency_seconds': round(latencies[-1], 3),
},
ensure_ascii=False,
)
)
if __name__ == '__main__':
run(main)