5 Commits
75 changed files with 7016 additions and 39 deletions
+6
View File
@@ -17,4 +17,10 @@ backups
__pycache__
node_modules
evidence
student-tests
tests
research
REPORT.md
SCHEMA.md
requirements-dev.txt
.DS_Store
+2
View File
@@ -18,3 +18,5 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
PUBLIC_PORT=8084
DATABASE_READ_AUTOCOMMIT=1
+12 -14
View File
@@ -1,18 +1,16 @@
## Лабораторная 4
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Горбанюк / БВТ2301.
Целевая оценка: 5. Трек: 2 — согласованность двух хранилищ.
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Реализация и воспроизводимая последовательность запуска: [REPORT.md](../REPORT.md).
Исходники приложения — src/, собственные проверки — student-tests/, результаты — evidence/.
Сохранено 65 собственных проверочных случаев; условия и результаты конкретных запусков приведены в отчёте.
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Контракт, исходное задание и публичные тесты сохранены. Секреты создаются локально и не входят в Git.
Ограничения и использованные первичные источники перечислены в REPORT.md.
- [x] Реализован обязательный минимум и заявленный трек.
- [x] Приведены команды запуска, проверок и безопасной остановки.
- [x] Сохранены результаты проверок и ограничения выводов.
- [x] Credentials и приватные ключи находятся вне репозитория.
+22
View File
@@ -0,0 +1,22 @@
name: Application integration
on: [push, pull_request]
permissions:
contents: read
jobs:
application:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.14.7'
- run: python -m pip install uv==0.12.13
- run: uv sync --locked
- run: make bootstrap lint check
- run: VCS_REF=$(git rev-parse HEAD) docker compose --env-file .env -f compose.yaml -f compose.test.yaml up -d --build --wait
- run: PYTEST_ADDOPTS=--disable-warnings make test-student
- run: make test
- run: uv run --locked python scripts/verify_cluster.py --output /tmp/cluster-ci.json
- if: always()
run: docker compose --env-file .env -f compose.yaml -f compose.test.yaml down
+1
View File
@@ -0,0 +1 @@
3.14.7
+29
View File
@@ -0,0 +1,29 @@
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56 AS builder
WORKDIR /app
RUN python -m pip install --no-cache-dir uv==0.12.13
ENV UV_PYTHON_DOWNLOADS=never
COPY pyproject.toml uv.lock .python-version ./
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-install-project --compile-bytecode
COPY src ./src
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev --no-editable --compile-bytecode
FROM python:3.14.7-slim-bookworm@sha256:82bc3c539b8813ada9d68c63b40158fa002f7f33de9bf3312a3dfdc0620dff56
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
WORKDIR /app
COPY --from=builder /app/.venv /app/.venv
COPY migrations /app/migrations
ARG VCS_REF=uncommitted
LABEL org.opencontainers.image.revision=$VCS_REF
ENV PATH="/app/.venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
APP_LOG_LEVEL=INFO
USER 10001:10001
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ.get('APP_PORT', '8080') + '/health/ready', timeout=1).close()"]
CMD ["python", "-m", "messenger"]
+19 -1
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://127.0.0.1:8084
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -28,3 +28,21 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: bootstrap up down image lint test-student verify reconcile
bootstrap:
$(PYTHON) scripts/bootstrap.py
up:
VCS_REF=$$(git rev-parse HEAD) docker compose --env-file .env up -d --build --wait
down:
docker compose --env-file .env down
image:
VCS_REF=$$(git rev-parse HEAD) docker compose --env-file .env build
lint:
uv run --locked ruff check src student-tests scripts/bootstrap.py scripts/cluster.py scripts/stop-one.py scripts/rolling-update.py scripts/verify_cluster.py scripts/verify_storage.py scripts/storage_lab.py research
test-student:
uv run --locked pytest --disable-warnings
verify:
uv run --locked python scripts/verify_cluster.py
reconcile:
docker compose --env-file .env run --rm reconciler
+374 -24
View File
@@ -1,37 +1,387 @@
# Отчёт — лабораторная 4
# Лабораторная 4 — Горбанюк, БВТ2301
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
## Запуск из чистого clone
Целевая оценка: **5**, трек 2 «Согласованность двух хранилищ». Работа подготовлена
для проверки. Название МР:
`ЛР 4 — БВТ2301 — Горбанюк — на 5`. Проверенная версия отмечается `lab4-verified`.
Окончательную оценку выставляет преподаватель, публичный smoke — только минимум.
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Перенесена собственная реализация lab3: слои HTTP/domain/service/repository,
сессии PostgreSQL, Argon2id, кэш Valkey, keyset-пагинация, две API за HAProxy,
идемпотентные сообщения, отдельный migrate, action hooks. README, COURSE,
CONTRIBUTING, contracts и публичные tests именно lab4 сохранены без изменений.
Все заявленные результаты подтверждаются командами и evidence.
## Архитектура и решения
Публичный форк: <https://gitea.privcloud.tech/Wemiliam/mtusi2026lab4>. Ветка решения: `solution/bvt2301-gorbanyuk`, целевая ветка преподавателя — `main`. Ветка опубликована; МР преподавателю: <https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/36>.
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
## Запуск из отдельного чистого clone
## Доказательства критериев
Нужны Docker Engine/Desktop + Compose v2, Git, Python 3.14.7 и uv 0.12.13.
Go на host не нужен: S3-сервер и клиент собираются внутри Docker. Первая сборка
требует доступа к реестрам, Python packages и Go modules и занимает заметное время.
Повторные сборки используют слои/cache. Нужны свободные RAM/диск для компиляции
и контейнеров; испытанный Docker-host: ARM64, 18 CPU, около 7.75 GiB RAM.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
```sh
make bootstrap
uv sync --locked
make check
uv run --locked python scripts/validate_openapi.py
make up
make test
```
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Адрес клиента: **http://127.0.0.1:8084**, только локальные синтетические данные.
bootstrap создаёт новый случайный пароль PostgreSQL и отдельные S3 credentials
для root/API/reconciler/tests в игнорируемом secrets/, создаёт .env; существующие
значения не заменяет. Каталог 0700, файлы 0644 для non-root Compose secret mounts.
Не переносить чужие .env/secrets из lab3: у этой работы собственные volumes/проект.
## Эксперименты и отказы
`make up` собирает образы, поднимает PostgreSQL и S3, выполняет отдельные migrate
и bucket-init, затем API ×2/Valkey/HAProxy и ждёт readiness. Bucket-init идемпотентен:
создание bucket с ignore-existing, запрет anonymous, повторное применение scoped
policies/users с теми же локальными ключами. Все зависимости поднимаются здесь.
Ни S3 API/console, ни PostgreSQL/Valkey, ни API напрямую в обычном compose
не публикуются; единственный host-port — loopback HAProxy.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Для собственных тестов используется override. Он открывает только loopback
PostgreSQL 55435, Valkey 56382 и S3 API 59004 и создаёт отдельные test bucket/identity:
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
```sh
docker compose --env-file .env -f compose.yaml -f compose.test.yaml up -d --build --wait
make test-student
make lint
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
# Вернуть остановленную реплику:
docker compose --env-file .env up -d --no-build --wait
make test-persistence ACTION_SCRIPT=scripts/restart.sh
```
## Ограничения и следующие шаги
Тестовая БД создаётся/удаляется отдельно; test bucket не совпадает с основным.
API основной системы не имеет credentials тестового bucket. Не запускать
разрушительные ручные тесты/эксперименты против данных вне этого учебного проекта.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Полная проверка реальных контейнеров создаёт новые синтетические объекты:
```sh
make verify
```
По умолчанию получается новое evidence/cluster-<timestamp>.json. Для сохранённого
запуска использовалось `--output evidence/cluster.json`. Проверяется stop каждой
API по очереди, реальный CID/PID, сообщения, rolling replacement и полный restart
включая S3. Восстанавливается остановленная реплика. Hook stop-one.sh требует
TARGET_INSTANCE_ID, публичный resilience.py передаёт его сам.
Обычная остановка сохраняет PostgreSQL/S3 volumes:
```sh
make down
```
Намеренно удалить **только учебные данные lab4**, если они больше не нужны:
```sh
docker compose --env-file .env down --volumes
```
Скрипты не используют общий Docker prune и не останавливают другие лабораторные.
Дополнительно проверен отдельный чистый clone коммита `900182c`: новые credentials,
новые PostgreSQL/S3 volumes, сборка образа с OCI revision этого коммита, все 65
тестов, публичные smoke и persistence. Хеши всех runtime-модулей в образе совпали
с исходниками clone и версией измерений. Временный стенд и только его новые
volumes удалены; основные данные lab4 сохранены. Доказательства:
[clean-clone.txt](evidence/clean-clone.txt), [clean-clone.json](evidence/clean-clone.json).
Workflow application.yml подготовлен, удалённый runner здесь не запускался.
## Архитектура и версии
```mermaid
flowchart LR
C[Client] --> LB[HAProxy :8084]
LB --> A1[API 1]
LB --> A2[API 2]
A1 --> PG[(PostgreSQL: metadata, sessions, messages)]
A2 --> PG
A1 --> S3[(Private S3: original bytes)]
A2 --> S3
A1 --> V[Valkey: disposable User cache]
A2 --> V
R[Reconciler: dry-run/apply] --> PG
R --> S3
```
Детальная схема/протокол и ограничения — [SCHEMA.md](SCHEMA.md). Новая миграция 003:
Attachment, внутренний storage_state, связи с message; составные FK запрещают
вложение из другого чата. Сервер назначает UUID key originals/<UUID>, filename
остаётся отображаемой строкой. Идентификация/ACL проверяются при upload, metadata,
content и отправке сообщения. Участник может прикрепить вложение другого участника
этого же чата. Посторонний получает 404. Публичный ответ не раскрывает object key.
Используется реляционная модель для связей/транзакций, объектное хранилище для
байтов, stateless API и явный протокол восстановления. PostgreSQL — источник
сессий, членства и результатов повторов. Кэш не даёт право доступа. Sticky
sessions нет. Идемпотентность сообщения включает text и упорядоченные attachment_ids;
отсутствующий список равен []. Message + links + результат ключа коммитятся вместе.
Версии: Python 3.14.7 (digest в Dockerfile), FastAPI 0.143.3, Pydantic 2.14.1,
psycopg 3.3.6, Argon2id через argon2-cffi 25.1.0, boto3 1.43.109, Pillow 13.0.0.
Точные транзитивные версии/hashes — uv.lock. PostgreSQL 18.3, Valkey 8.1.3-alpine,
HAProxy 3.2.1-alpine. Образы S3 собираются из официального исходного релиза MinIO
**RELEASE.2025-10-15T17-29-55Z** с исправлением CVE, mc — RELEASE.2025-08-13T08-35-41Z.
Готовые upstream Docker Hub теги оказались недоступны, поэтому используется
infra/s3.Dockerfile. Компилятор Go 1.26.7-alpine3.23 и runtime Alpine 3.23 закреплены
digest. Community-репозиторий MinIO архивный: это воспроизводимая учебная версия,
поддержка production здесь не заявляется. Go module source/tag и локальный image ID
фиксируются в доказательствах, а не подменяются обещанием latest.
Собранный через go install бинарник сообщает DEVELOPMENT.GOGET; это build label,
а точная исходная pseudo-version/commit и SumDB hash извлечены из Go build info
в [s3-source-version.txt](evidence/s3-source-version.txt).
Миграции выполняются одной отдельной службой под advisory lock с checksum.
API сам DDL не запускает. Миграции 001/002 не менялись. 003 добавляет схему вложений,
но внешние wire-ответы lab4 изменились: старые lab3 API нельзя смешивать с lab4 за LB.
Rolling здесь — последовательное пересоздание процессов тем же образом, с
ожиданием готовности между шагами, не произвольное обновление бизнес-схемы.
## Поток, безопасность и ограничение ресурсов
Upload — raw PNG/JPEG, не multipart. Auth/ACL, filename и Content-Length проверяются
до чтения body. Опасные разделители, NUL/управляющие символы/суррогаты отклоняются.
Внешний MIME должен совпасть с PNG/JPEG, обнаруженным Pillow. До полной распаковки
проверяется площадь; verify и повторное load отсекают повреждённый поток. Байты
никогда не перекодируются, SHA-256 считается инкрементально.
На API одновременно обрабатывается **один upload**, остальные ждут до 5 s,
потом 429. Сам поток ограничен 15 s и фактическими **10 485 760 bytes**; объявленному
Content-Length без проверки не доверяем. **40 000 000 пикселей** проверяются до load.
Анонимный временный файл находится в tmpfs 32 MiB и закрывается при успехе,
ошибке или отключении клиента. Это scratch, от него не зависит доступность файла
после 201. RAM декодера зависит от ограниченной площади, не только сжатых bytes;
один слот не даёт десяти декодерам одновременно занять память одной API.
Boto3 выполняет один PutObject ≤10 MiB: multipart transfer/автоматических retries
нет, connect timeout 0.5 s, socket read timeout 2 s. Повтор сообщения защищён
Idempotency-Key, повтор upload может создать новое вложение. SDK/SQL/декодирование
выполняются вне event loop. Download передаёт 64 KiB chunks через API с ACL,
Content-Length, истинным Content-Type, безопасным Content-Disposition и nosniff;
redirect в S3 нет. Клиент проверяет итоговый SHA. Разрыв stream после начала ответа
нельзя превратить в JSON-ошибку задним числом: такой GET нужно повторить.
API: 1 CPU / 512 MiB, readonly filesystem, UID 10001, cap_drop ALL, tmpfs 32 MiB,
один Uvicorn worker. PostgreSQL: 1 CPU / 768 MiB; S3: 1 CPU / 768 MiB;
Valkey: 0.5 CPU / 128 MiB; LB: 0.5 CPU / 128 MiB. API swap сверх лимита запрещён.
API не публикует порт и не имеет container_name. Пулы/SQL timeout сохранены из lab3.
SIGTERM: Uvicorn перестаёт принимать соединения, grace 5 s, Compose ждёт 7 s.
Не подтвердившийся upload оставляет pending, который восстановит сверка.
HAProxy: round-robin, readiness каждые 500 ms, fall 2/rise 1, connect/check 500 ms,
Docker DNS обновляется с hold valid 1 s; адреса не зашиты. LB client timeout 20 s,
server timeout 30 s учитывают bounded upload. Readiness требует PostgreSQL и S3;
при отказе S3 LB может временно исключить все API. Valkey не критичен.
Приватность/права: API identity умеет Get/Put только в originals/ основного bucket,
List/GetLocation только этого bucket для readiness. Нет DeleteObject, bucket/admin
операций или доступа к чужому bucket. Reconciler имеет List/Get/Delete своего
prefix, без Put. Root передан только S3/bootstrap. Консоль привязана к loopback
внутри S3-контейнера и не опубликована. Отдельный test identity ограничен test bucket.
Secret values отсутствуют в Git, образах, evidence и логах; локальные secret mounts
не заменяют защиту Docker-host и secret manager удалённого окружения.
## Согласованность и сверка: трек 2
201 возвращается после PUT и COMMIT ready. До PUT уже есть устойчивый pending.
Это не двухфазный COMMIT между S3 и PostgreSQL. При отключении клиента до конца
body не создаются ни intent, ни S3 объект; после intent любой неясный исход остаётся
учтённым. Сверка отдельно восстанавливает запись или удаляет брошенное намерение.
```sh
make reconcile # только dry-run
# Изменения включаются явно:
docker compose --env-file .env run --rm reconciler python -m messenger.reconcile --apply --grace-seconds 300
```
Grace не может быть меньше 300 s. CLI работает постранично (S3 до 1000 объектов,
PostgreSQL до 100 строк), preview до 100 actions, counters полные. Один reconciler
на БД благодаря advisory lock, исправляемые строки блокируются FOR UPDATE.
| Обнаружено | Безопасное исправление после grace |
| --- | --- |
| Старый orphan нашего UUID-prefix без metadata | Повторный HEAD, проверка ETag/size/возраста, DELETE только этого объекта |
| Старый pending, PUT сохранил правильный объект | ready; checksum/size/MIME сверяются с намерением |
| Старый pending, подтверждённое 404 объекта, нет links | Удаление брошенного намерения |
| Ready, объект пропал/не соответствует metadata | missing, metadata и Message links сохраняются; API выдаёт 409 |
| Missing, правильный оригинал восстановлен оператором | ready, тот же ID и связи |
| Свежие объекты/намерения, ready с/без Message links | Не удаляются |
| Сетевой сбой, 403, 5xx | Проход завершается ошибкой, это не отсутствие объекта |
| Неожиданные bytes у pending | Review action, автоматическое удаление запрещено |
S3 LIST и HEAD имеют разную точность даты. Проверяется возраст обеих наблюдаемых
записей и ETag/size, а не точное равенство LastModified. ETag используется как
проверка неизменности, не как SHA оригинала. Приложение не переиспользует ключи.
Сторонний writer с правом overwrite не должен менять originals/ во время сверки.
В dry-run нет изменений данных. Повтор apply сходится к отсутствию новых actions.
При ошибке часть независимых исправлений могла уже закоммититься; запуск можно
повторить. Потерянные bytes нельзя восстановить из SHA: для фактического восстановления
нужна исходная копия/backup. Сверка проверяет HEAD metadata, а не перехеширует весь
bucket; сквозные проверки отдельно скачивают оригинал и считают SHA.
## Команды исследований и критерии
Для проверки настоящего orphan нужен его **реальный** возраст ≥300 s, часы S3
не подменяются. Сначала создать seed, затем выполнять прочие проверки:
```sh
uv run --locked python scripts/verify_storage.py --seed evidence/storage-seed-new.json
make test-student
make verify
# Когда LastModified seed старше 300 s:
uv run --locked python scripts/verify_storage.py --seed-file evidence/storage-seed-new.json --output evidence/storage-new.json
uv run --locked --group research python research/analyze.py evidence/storage-new.json
```
Сохранённый запуск разбит на базовые проверки и продолжение после aging
(`--resume-basics evidence/storage-basics.json`); memory/cost повторены отдельно
на финальном образе без параллельного запуска тестов. Итоговый набор — storage.json.
Скрипт отказывается перезаписывать output. Для графиков сохранённой серии
analyze.py читает evidence/storage.json. Raw содержит cgroup readings, времена
каждой загрузки, CID/alias и counters/время всех проходов сверки, без токенов.
| Критерий | Проверка | Доказательство |
| --- | --- | --- |
| Исходный skeleton/OpenAPI | make check; validate_openapi | [contracts.txt](evidence/contracts.txt) |
| Весь API lab4 и прежние возможности | make test; make test-student | [smoke.txt](evidence/smoke.txt), [student-tests.txt](evidence/student-tests.txt) |
| PNG/JPEG, MIME, лимиты, ACL, чужой chat/id | 65 собственных тестов с настоящими PG/S3 | [test_attachments.py](student-tests/test_attachments.py) |
| Байты с другой API после stop загрузившей | cross_replica в verify_storage | [storage.json](evidence/storage.json) |
| Сохранность metadata/bytes/session при restart | публичный persistence, StartedAt PG/Valkey/S3/LB | [persistence.txt](evidence/persistence.txt), [cluster.json](evidence/cluster.json) |
| Каждая API, идемпотентность, rolling | make verify | [cluster.json](evidence/cluster.json) |
| Приватный bucket и ограниченный API user | анонимный GET 403, Delete/other bucket 403 | [storage.json](evidence/storage.json) |
| Прерванный body / отказ S3 | raw TCP abort; реальный stop/start S3 | [storage.json](evidence/storage.json) |
| PUT прошёл, DB finalize упал; PUT не прошёл | injected failure между шагами, dry-run/apply/repeat | [test_attachments.py](student-tests/test_attachments.py) |
| ≥10 загрузок, bounded RAM | барьер 10 клиентов, два размера, kernel memory.peak | [storage.json](evidence/storage.json) |
| Orphan/missing, свежие/used защищены | реальный aged orphan, удаление used bytes, восстановление | [storage.json](evidence/storage.json) |
| Стоимость обхода | 0/100/1100 дополнительных объектов, 3 dry-run повтора | [storage.json](evidence/storage.json) |
## Наблюдаемые результаты
65 собственных тестов и публичный smoke прошли. Публичные persistence/failover
сохранили прежнюю сессию и оригинал; реальный S3 stop дал два 503, после start
тот же файл снова скачался. Консоль недоступна из API, повторный bucket bootstrap
дважды завершился успешно с теми же credentials и данными.
| Серия загрузок | Клиенты | Успешные RPS | p50/p95/p99, ms | Максимум memory.peak одной API, MiB | Ошибки |
| --- | ---: | ---: | --- | ---: | ---: |
| 1,081,015 bytes, 360,000 pixels | 10 | 58.07 | 121.6/159.4/159.4 | 162.77 | 0 |
| 8,674,572 bytes, 2,890,000 pixels | 10 | 13.52 | 500.3/726.9/726.9 | 163.62 | 0 |
Средняя суммарная загрузка CPU двух API по delta cgroup cpu.stat: 0.58 ядра, 0.91 ядра
для малого/большого payload. Окно включает probes чтения счётчиков и служебные процессы после readiness; это короткая усреднённая оценка, не CPU p95. Raw cpu.stat
сохранён в before/after каждой серии.
Kernel peak около 163 MiB при лимите 512 MiB. Он включает startup (в том числе
Argon2id) и является верхней границей за весь trial. Из близости двух peak
не следует, что размер файла не влияет на RAM: эксперимент не отделяет пик
startup от пика декодирования. Файлы больше проверенных размеров, но в пределах
10 MiB/40 млн pixels, покрыты лимитами реализации; их полная матрица RAM не измерялась.
| Дополнительные fresh S3 objects | dry-run min / median / max, ms | LIST pages |
| ---: | --- | --- |
| 0 | 137.78 / 141.20 / 184.41 | 1,1,1 |
| 100 | 150.60 / 163.01 / 166.15 | 1,1,1 |
| 1100 | 189.81 / 216.18 / 233.53 | 2,2,2 |
Настоящий orphan был выдержан 503.0 s. Dry-run обнаружил
его и брошенный intent, но не изменил S3/БД; apply удалил ровно эти объекты,
повторный apply вернул пустой actions. Свежий orphan сохранён. Для used-вложения
после удаления bytes dry-run показал mark_missing без изменения metadata;
apply сохранил ID и Message link и перевёл запись в missing. После возврата
правильных bytes следующий проход вернул ready, download снова совпал с fixture.
Это подтверждённое улучшение протокола: сбой между двумя хранилищами больше не
оставляет незаметный вечный orphan/pending; он обнаруживается и сходится к явному
состоянию. Это не обещание восстановления без исходных bytes.
| Воздействие API | Успешные RPS | Ошибки/попытки | Успех после hook, s | p95, ms |
| --- | ---: | --- | ---: | ---: |
| Остановка API 1 | 421.2 | 167/2615 | 2.225 | 31.2 |
| Остановка API 2 | 334.3 | 153/1964 | 1.819 | 44.8 |
| Rolling replacement | 432.3 | 287/7903 | 13.610 | 47.9 |
Каждая остановка укладывается в 10 s; подтверждённые сообщения сохранены.
first_success_after_action_seconds отсчитывается от старта hook до первого
успеха после его завершения, это не длительность сплошного простоя. Другая API
может отвечать в течение hook. В stopped хранится паспорт процесса до stop;
скрипт отдельно проверяет running=false. Проверка cross_replica сохраняет также
фактический after_stop с running=false и alias другой скачавшей API.
![RAM загрузок и стоимость сверки](evidence/storage-research.png)
CPU/RAM лимиты проверялись inspect. Kernel memory.peak включает Python, decoder,
tmpfs, page cache и startup; это не process RSS и не усреднение Docker stats.
Каждый размер испытывался одной серией из 10 клиентов, API пересоздавались до
серии. Десять клиентов не означают десять одновременных декодеров: именно gate
ограничивает активную обработку, остальные ждут, все десять завершились успешно.
Короткие локальные измерения не являются production capacity/SLO.
Стоимость сверки измерена при неизменном числе metadata и росте числа свежих
S3 objects. Это проверяет paging и постоянные накладные расходы; масштабирование
по числу metadata отдельно не измерялось. LIST стоит O(objects/1000), индексированный
DB lookup нужен для старых объектов, HEAD — для старых metadata. Нет вывода, будто
полное чтение/хеширование всех bytes столь же дёшево. Три повтора дают min/median/max,
не статистический доверительный интервал. Обе API и health probes работают во время
замеров, остальные тесты/сборки не запускаются.
## Вопросы на защите
**Почему Docker volume одной API не равен S3?** Локальный файл зависит от выбранного
host/контейнера и не появляется у соседней реплики. S3 — общий адресуемый сервис
с object key, API хранения и отдельными правами. Временный tmpfs здесь нужен только
во время запроса; после 201 оригинал читается из общего S3, что проверено остановкой
uploading process. S3 volume тоже нуждается в защите от отказа диска/host.
**Когда честно вернуть 201?** Когда bytes полностью приняты, проверены, PUT успешно
подтверждён и metadata ready закоммичена. Возвращать 201 после помещения в локальную
память нельзя. В lab5 будет другой контракт: 202 после устойчивого staging и
надёжного сохранения намерения обработки; ready появится позже.
**PUT прошёл, а COMMIT — нет?** Два хранилища не откатываются одной SQL-транзакцией.
До PUT уже сохранён pending, поэтому объект не теряется из учёта. После grace
reconciler сверит key/checksum/size/MIME и восстановит ready. Если Put не дошёл,
старый пустой intent удалится. Если metadata вообще нет, старый orphan удаляется
после дополнительной проверки. HTTP timeout сам по себе не доказывает rollback.
**Почему случайный key/private bucket не заменяют ACL?** Случайность — защита от
угадывания, приватность — от анонимного S3. Любой участник приложения всё равно
может знать ID чужого вложения. Только проверка действительной сессии и членства
в конкретном чате на metadata/content/message запрещает чужой доступ. Автор
выводится из сессии, filename/key не используются как доказательство прав.
## Ограничения и продолжение
Защищён отказ одной API, но host/LB/PostgreSQL/MinIO и их диски остаются SPOF.
Backup/PITR/репликация storage не заявляются. Это локальный HTTP-стенд с вымышленными
данными; удалённая демонстрация потребует TLS. Presigned upload (трек 1) не реализован.
Готовые неиспользованные вложения сохраняются; политики удаления истории нет.
Для неожиданного несовпадения pending требуется оператор, оно явно видно в report.
Reconciler запускается вручную по команде; для регулярной эксплуатации его нужно
запускать по расписанию, вечное автоматическое исправление без запуска не обещается.
В lab5 появятся устойчивое staging, outbox/job, брокер и worker. Объекты очереди
будут ссылками, не bytes. Потребуются lease/retry и защита активной обработки от GC;
обязательный thumbnail будет отдельным вариантом. Lab4 останется независимым
проверенным репозиторием, его контракт 201 ready не меняется будущей работой.
## Источники
- [README](README.md), [семантика](contracts/README.md), [COURSE](COURSE.md).
- [Официальный релиз MinIO с исправлением CVE](https://github.com/minio/minio/releases/tag/RELEASE.2025-10-15T17-29-55Z).
- [MinIO policy access](https://docs.min.io/aistor/administration/iam/access/).
- [Pillow Image API](https://pillow.readthedocs.io/en/stable/reference/Image.html): lazy open, verify, pixel guard.
- [Boto3 retries](https://docs.aws.amazon.com/boto3/latest/guide/retries.html).
- [AWS S3 object uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html).
- [Linux cgroup v2 memory interface](https://docs.kernel.org/admin-guide/cgroup-v2.html): memory.current/peak/max.
## Соответствие ревизий
После редактуры отчёта, шаблона и служебных метаданных сохранено соответствие прежних и текущих Git-ревизий: [revision-map.json](evidence/revision-map.json). Код приложения, миграции, зависимости, Dockerfile, конфигурация сервисов и исходные публичные проверки не изменены. Численные результаты и исторические image/source identifiers в evidence сохранены в исходном виде; их не следует трактовать как новый запуск измерений.
+75
View File
@@ -0,0 +1,75 @@
# Lab4: метаданные, байты и восстановление
Основа — собственная lab3. PostgreSQL хранит users/chats/sessions/messages и
результаты Idempotency-Key. Valkey — восстанавливаемый кэш User. В lab4 добавлены
Attachment и message_attachments; оригиналы находятся в приватном S3.
Миграции 001/002 перенесены без изменения checksum. 003 допускает пустой text
для сообщения с вложением и добавляет таблицы. Обязательность содержимого
проверяет модель HTTP; составные FK в message_attachments гарантируют один chat
у сообщения и вложения. Position сохраняет порядок ID. До десяти уникальных ID,
проверка доступности и вставка связей входят в транзакцию message + request result.
Для текстового POST отсутствие attachment_ids и [] дают один hash.
Attachment имеет id/chat/owner, отображаемый filename, фактический MIME,
size_bytes, SHA-256, серверный object_key, storage_state и timestamps.
Object key: originals/<UUID>; имена не используются как пути.
Storage_state — внутреннее состояние pending/ready/missing. Внешняя схема lab4
позволяет только ready; для pending/missing API возвращает 409, а не выдумывает
несуществующий статус. Чужой чат — 404, перед любым GET выполняется ACL.
## Загрузка
1. Auth/ACL и filename/MIME/Content-Length проверяются до чтения файла.
2. Один активный upload на процесс. До входа очередь ждёт максимум 5 s.
3. Request.stream читается с общим бюджетом 15 s и счётчиком фактических bytes.
Временный анонимный файл — только ограниченная scratch-копия на tmpfs 32 MiB.
4. Pillow читает заголовок и проверяет площадь до декодирования; verify + повторное
load отклоняют повреждённые PNG/JPEG. Байты не перекодируются. SHA считается
инкрементально по исходному потоку. Размер ≤10 MiB, площадь ≤40 млн пикселей.
5. COMMIT записи pending с известными key/checksum **до** S3 PUT.
6. PUT целого объекта; затем отдельный COMMIT перевода в ready; только затем 201.
Падение до шага 5 не создаёт объектов. После 5 любой исход PUT/COMMIT остаётся
связан с устойчивым намерением. Состояние 201 не кэшируется до COMMIT.
Повтор upload может создать новое вложение: Idempotency-Key гарантирует повтор
сообщения, а не бинарного POST. Прямой загрузки и multipart upload здесь нет.
## Сверка
Отдельный CLI с отдельным S3 identity; dry-run по умолчанию. --apply разрешает
исправления. Минимальный grace — 300 s, заметно больше штатной загрузки/остановки.
Ключи не переиспользуются и не перезаписываются приложением. Сверка берёт advisory
lock на весь запуск, обрабатывает списки постранично, строки — под FOR UPDATE.
- Orphan: старый объект строго нашего UUID-prefix без строки Attachment.
Повторный HEAD проверяет ETag, размер и возраст непосредственно перед DELETE.
LIST и HEAD имеют разную точность timestamp, точное равенство дат не требуется.
Свежий объект пропускается, другие prefix/buckets не затрагиваются.
- Старый pending + существующий совпадающий объект → ready. Это восстановление
случая «PUT завершён, финальный COMMIT/HTTP не подтверждён».
- Старый pending + подтверждённое отсутствие объекта → удаление намерения,
только если нет message relation. Свежие намерения защищены.
- Старый ready + отсутствующие/несогласованные bytes → missing; строка и ссылки
сохраняются. GET возвращает 409. Сверка не способна восстановить утраченные байты
из SHA; после восстановления правильного оригинала она возвращает ready.
- Ready-объекты, даже не прикреплённые к сообщению, не удаляются. Действующие ссылки
никогда не удаляются. Ошибки сети/403/5xx не трактуются как 404.
- Несогласованный pending с чужими checksum/size/MIME требует оператора;
report содержит review_invalid_pending. Автоматически удалять такие bytes нельзя.
Сравнение HeadObject проверяет size/MIME/метаданные SHA, а не повторно хеширует
все bytes; полная проверка SHA выполняется клиентскими сценариями скачивания.
Обычный проход: LIST O(objects/1000), индексированный lookup для старых объектов,
HEAD для старых metadata, O(metadata) строк. Память ограничена страницами и
100 записями preview, counters учитывают весь проход. При ошибке часть apply
могла уже завершиться: повторный запуск безопасен и доводит сверку до результата.
## Границы гарантий
Это протокол восстановления двух независимых хранилищ, не распределённая ACID
транзакция. PostgreSQL/S3/диск/LB и host остаются SPOF. Глобальные backup/PITR не
реализованы. Clock skew предполагается значительно меньше grace. Поддерживаются
только серверные неизменяемые UUID-ключи; сторонний writer с правом перезаписи
не должен менять их во время сверки. Для lab5 придётся учесть lease активного worker
и staging-prefix: этот CLI не должен удалять объекты по ещё активному заданию.
+11
View File
@@ -0,0 +1,11 @@
# Only DB/cache test ports; API replicas always stay behind the LB.
services:
postgres:
ports: ["127.0.0.1:${TEST_PG_PORT:-55435}:5432"]
valkey:
ports: ["127.0.0.1:${TEST_CACHE_PORT:-56382}:6379"]
s3:
ports: ["127.0.0.1:${TEST_S3_PORT:-59004}:9000"]
bucket-init:
environment:
CREATE_TEST_BUCKET: '1'
+194
View File
@@ -0,0 +1,194 @@
name: mtusi-lab4
x-api: &api
image: ${API_IMAGE:-mtusi-messenger:lab4}
build:
context: .
args:
VCS_REF: ${VCS_REF:-uncommitted}
user: "10001:10001"
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
pids_limit: 96
cpus: ${API_CPU:-1}
mem_limit: ${API_MEMORY:-512m}
memswap_limit: ${API_MEMORY:-512m}
stop_grace_period: 7s
secrets:
- database_url
- cache_url
- source: s3_api_access_key_id
target: s3_access_key_id
- source: s3_api_secret_access_key
target: s3_secret_access_key
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
APP_ENV: lab4
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DATABASE_READ_AUTOCOMMIT: ${DATABASE_READ_AUTOCOMMIT:-1}
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-8}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
networks: [backend]
services:
postgres:
image: postgres:18.3
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql]
command: [postgres, -c, fsync=on, -c, synchronous_commit=on, -c, full_page_writes=on,
-c, shared_preload_libraries=pg_stat_statements]
healthcheck:
test: [CMD-SHELL, "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 30
cpus: ${POSTGRES_CPU:-1}
mem_limit: 768m
memswap_limit: 768m
networks: [backend]
valkey:
image: valkey/valkey:8.1.3-alpine
command: [valkey-server, --save, "", --appendonly, "no", --maxmemory, 64mb,
--maxmemory-policy, allkeys-lru]
healthcheck:
test: [CMD, valkey-cli, ping]
interval: 2s
timeout: 1s
retries: 20
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
networks: [backend]
migrate:
<<: *api
command: [python, -m, messenger.migrate]
secrets: [database_url]
healthcheck: {disable: true}
depends_on:
postgres: {condition: service_healthy}
api:
<<: *api
deploy:
replicas: ${API_REPLICAS:-2}
depends_on:
migrate: {condition: service_completed_successfully}
valkey: {condition: service_healthy}
bucket-init: {condition: service_completed_successfully}
lb:
image: haproxy:3.2.1-alpine
volumes: [./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
ports: ["127.0.0.1:${PUBLIC_PORT:-8084}:8080"]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=4m"]
cap_drop: [ALL]
security_opt: [no-new-privileges:true]
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 2s
timeout: 2s
retries: 30
depends_on:
api: {condition: service_healthy}
networks: [backend]
s3:
image: mtusi-lab4-s3:2025-10-15
build: {context: ., dockerfile: infra/s3.Dockerfile, target: server}
command: [server, /data, --address, ':9000', --console-address, '127.0.0.1:9001']
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=32m"]
cap_drop: [ALL]
cpus: 1
mem_limit: 768m
memswap_limit: 768m
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:9000/minio/health/live']
interval: 2s
timeout: 1s
retries: 30
networks: [backend]
bucket-init:
image: mtusi-lab4-mc:2025-08-13
build: {context: ., dockerfile: infra/s3.Dockerfile, target: client}
command: [/bootstrap-s3.sh]
volumes:
- ./infra/bootstrap-s3.sh:/bootstrap-s3.sh:ro
- ./infra/api-policy.json:/policies/api-policy.json:ro
- ./infra/reconciler-policy.json:/policies/reconciler-policy.json:ro
secrets: [s3_root_user, s3_root_password, s3_api_access_key_id, s3_api_secret_access_key,
s3_reconciler_access_key_id, s3_reconciler_secret_access_key,
s3_test_access_key_id, s3_test_secret_access_key]
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
read_only: true
tmpfs: ["/tmp:rw,noexec,nosuid,size=16m"]
cpus: 0.5
mem_limit: 128m
networks: [backend]
depends_on:
s3: {condition: service_healthy}
reconciler:
<<: *api
profiles: [tools]
command: [python, -m, messenger.reconcile]
healthcheck: {disable: true}
secrets:
- database_url
- cache_url
- source: s3_reconciler_access_key_id
target: s3_access_key_id
- source: s3_reconciler_secret_access_key
target: s3_secret_access_key
depends_on:
migrate: {condition: service_completed_successfully}
bucket-init: {condition: service_completed_successfully}
networks:
backend: {}
volumes:
s3_data:
postgres_data:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_api_access_key_id:
file: ./secrets/s3_api_access_key_id
s3_api_secret_access_key:
file: ./secrets/s3_api_secret_access_key
s3_reconciler_access_key_id:
file: ./secrets/s3_reconciler_access_key_id
s3_reconciler_secret_access_key:
file: ./secrets/s3_reconciler_secret_access_key
s3_test_access_key_id:
file: ./secrets/s3_test_access_key_id
s3_test_secret_access_key:
file: ./secrets/s3_test_secret_access_key
+1
View File
@@ -0,0 +1 @@
PASS: research figure generated from storage.json
+2
View File
@@ -0,0 +1,2 @@
PASS: 103 candidate/versioned files; 129 reachable objects; no requested names or local credentials.
PASS: instructor files unchanged, upstream push disabled.
+2
View File
@@ -0,0 +1,2 @@
PASS: private bucket and scoped identities ready
PASS: private bucket and scoped identities ready
+68
View File
@@ -0,0 +1,68 @@
{
"tested_commit": "900182cbb405d57b20c1b8bfd7cef21592ccf4cf",
"image_revision": "900182cbb405d57b20c1b8bfd7cef21592ccf4cf",
"runtime_sha256": {
"reconcile.py": "6c96d72a8c49a03ff721029545a41c3587bc24be3272df4003bf6141dd04df9c",
"cursor.py": "a2208cba232ae0f67c4430d3e90c3962129c8fb3987ea853c624cdd2c73d7a1f",
"cache.py": "17756fa84a2c40fd4d1baf9602a7b755ca5998bc241f0657db3505b04623321d",
"service.py": "7e100756e3bb24a89bce5dfa41a471969cdb4287c1523673d1e7ff8255bf3c19",
"__init__.py": "f798dc149d6bef46d697dccdaf6be9c8d08654f18161e42c99f548c30074f4ae",
"__main__.py": "486cd5bbb3f1a814de74357e938968d219070e845cf178761a7a23e87d003ea4",
"app.py": "6dd900a5ff7b2213ace024b55dfd7313195d1fb1257bab6067b16d2b4f515941",
"settings.py": "d21f4b624767b19c80f534e608de6de6071260b5e2d642a06e6a7aec0148c161",
"auth.py": "6bebcd8d1d3ca4c2c6e8b0e5793ae13d8996cac3709b5b845d814b51a5e1a3c3",
"attachments.py": "3fbf029536834be791d39b38fb76e2eb694b46a57d891ff73763b3a7be99234f",
"storage.py": "2fd071caf423a09378d8fc162dc5f675484addf8589e0e0228057c6b7bb77d75",
"migrate.py": "f073c080e13635a01482539dbb474b9b4f49bbf5ff66f483dbda4fbe1d1eb57f",
"schemas.py": "3f94cc536beec31ec7890331abbc736445a311d03b30b9b5726dd60c17c9f6a9",
"repository.py": "d29dabc547e0071b21f8b9d5371fc5753b89766f2c3e8ea8412cdcb8a8c33384",
"domain.py": "bd8456d7423a3e12c02a57ed90d1875fc5c7acdcfc0d26b94f9ead4855160298"
},
"runtime_matches_measurements": true,
"fresh_secrets": true,
"fresh_pg_and_s3_volumes": true,
"student_tests_passed": 65,
"public_smoke_passed": true,
"public_persistence_passed": true,
"api_instances": [
{
"id": "f43056b397aa3639d9fe0220734c61f2e66caa5d6ce4522c35a645933787204d",
"started_at": "2026-10-17T17:22:12.959003129Z",
"pid": 9135,
"running": true,
"image": "sha256:4c8bff2f05a461167d1921b16e64fbd46342daffcaed4f1e9486dadea0ea3416",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "c4a9f8fb8759467b83f84981872cc014"
},
{
"id": "ca95b52ce4b1976a36b4f0e1aa32a7310cacee932f4f8090b410cf957dc931c3",
"started_at": "2026-10-17T17:22:13.11128092Z",
"pid": 9269,
"running": true,
"image": "sha256:4c8bff2f05a461167d1921b16e64fbd46342daffcaed4f1e9486dadea0ea3416",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "8f7362c97d3f40c48a6c72dd5504fb1e"
}
],
"s3": {
"id": "3f7a14116acfd92ef834a5c71f2571b08c297e3786f16a7789bae5d28e4e7fae",
"started_at": "2026-10-17T17:22:12.732578545Z",
"pid": 8874,
"running": true,
"image": "sha256:ca2f62967cfd3096ef07615e356e3e6909bb6885e1672999fc0e206b478ec904",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59004"
}
]
}
}
}
+22
View File
@@ -0,0 +1,22 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
python3 tests/smoke.py --base-url "http://127.0.0.1:28084"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
uv run --locked pytest --disable-warnings
============================= test session starts ==============================
platform darwin -- Python 3.14.7, pytest-9.1.1, pluggy-1.7.0
rootdir: <repository>
configfile: pyproject.toml
testpaths: student-tests
plugins: anyio-4.15.1
collected 65 items
student-tests/test_api.py .................................... [ 55%]
student-tests/test_attachments.py ................. [ 81%]
student-tests/test_cluster.py ............ [100%]
======================= 65 passed, 2 warnings in 21.77s ========================
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:28084"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+307
View File
@@ -0,0 +1,307 @@
{
"initial_instances": [
{
"id": "79907dfdbd61808088ab2f17ab8cc4e5760c9dc8d20f38e726e830629e9a66bc",
"started_at": "2026-10-17T16:54:30.42283897Z",
"pid": 10590,
"running": true,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "fccb137fe0244ddf98a9e454eab19756"
},
{
"id": "47d2cb2718e3c28a5e08cf1932d3e715aa6df2fb7253e161adb1f139ed75da19",
"started_at": "2026-10-17T16:54:30.345976304Z",
"pid": 10534,
"running": true,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "1ee65c20c8df4f808677d83c6556e18d"
}
],
"failures": [
{
"duration_seconds": 5.81228291700063,
"workers": 6,
"requests": 2615,
"successes": 2448,
"errors": 167,
"successful_rps": 421.1770202788521,
"error_fraction": 0.06386233269598471,
"first_success_after_action_seconds": 2.225296332999278,
"action_seconds": 1.7712559159990633,
"p50_ms": 8.855707999828155,
"p95_ms": 31.16029100056039,
"p99_ms": 73.12783300039882,
"instances": [
"1ee65c20c8df4f808677d83c6556e18d",
"fccb137fe0244ddf98a9e454eab19756"
],
"tail_instances": [
"1ee65c20c8df4f808677d83c6556e18d"
],
"confirmed_writes": 243,
"stopped": {
"id": "79907dfdbd61808088ab2f17ab8cc4e5760c9dc8d20f38e726e830629e9a66bc",
"started_at": "2026-10-17T16:54:30.42283897Z",
"pid": 10590,
"running": true,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "fccb137fe0244ddf98a9e454eab19756"
}
},
{
"duration_seconds": 5.416828084000372,
"workers": 6,
"requests": 1964,
"successes": 1811,
"errors": 153,
"successful_rps": 334.32849850803495,
"error_fraction": 0.07790224032586558,
"first_success_after_action_seconds": 1.8193489170007524,
"action_seconds": 1.3782874589996936,
"p50_ms": 10.209916999883717,
"p95_ms": 44.78274999928544,
"p99_ms": 68.33825000103388,
"instances": [
"1ee65c20c8df4f808677d83c6556e18d",
"ec543faccabb4b4d856cb21490da98d9"
],
"tail_instances": [
"ec543faccabb4b4d856cb21490da98d9"
],
"confirmed_writes": 179,
"stopped": {
"id": "47d2cb2718e3c28a5e08cf1932d3e715aa6df2fb7253e161adb1f139ed75da19",
"started_at": "2026-10-17T16:54:30.345976304Z",
"pid": 10534,
"running": true,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "1ee65c20c8df4f808677d83c6556e18d"
}
}
],
"concurrent_replays": 30,
"concurrent_effects": 1,
"rolling": {
"duration_seconds": 17.618615958999726,
"workers": 6,
"requests": 7903,
"successes": 7616,
"errors": 287,
"successful_rps": 432.27004991329574,
"error_fraction": 0.03631532329495128,
"first_success_after_action_seconds": 13.610468083001251,
"action_seconds": 13.610214250000354,
"p50_ms": 9.198334000757313,
"p95_ms": 47.916500001520035,
"p99_ms": 80.57516699955158,
"instances": [
"156f31907e404f4b82cd2ece01102a69",
"5ae9d1ba363c48999bc872e0a1648271",
"c8354af5dac443e3afdfc970ba76d70e",
"ec543faccabb4b4d856cb21490da98d9"
],
"tail_instances": [
"156f31907e404f4b82cd2ece01102a69",
"5ae9d1ba363c48999bc872e0a1648271"
],
"confirmed_writes": 759,
"replacements": [
{
"old_container": "79907dfdbd61808088ab2f17ab8cc4e5760c9dc8d20f38e726e830629e9a66bc",
"old_instance": "ec543faccabb4b4d856cb21490da98d9",
"live_instances": [
"5ae9d1ba363c48999bc872e0a1648271",
"c8354af5dac443e3afdfc970ba76d70e"
]
},
{
"old_container": "47d2cb2718e3c28a5e08cf1932d3e715aa6df2fb7253e161adb1f139ed75da19",
"old_instance": "c8354af5dac443e3afdfc970ba76d70e",
"live_instances": [
"156f31907e404f4b82cd2ece01102a69",
"5ae9d1ba363c48999bc872e0a1648271"
]
}
],
"confirmed_messages_preserved": true
},
"persistence": {
"before": {
"postgres": {
"id": "a6e5f14597245ada8071635bf261c294e53045ee0d09bbea1cd682681cfacd9c",
"started_at": "2026-10-17T16:54:25.66952526Z",
"pid": 9916,
"running": true,
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"5432/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "55435"
}
]
}
},
"valkey": {
"id": "1899ba79d40145cd3ff95269e73076e3b2ba74a472e4427f204da8e9c52592b6",
"started_at": "2026-10-17T16:54:25.484294468Z",
"pid": 9764,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56382"
}
]
}
},
"s3": {
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
"started_at": "2026-10-17T16:54:25.588696677Z",
"pid": 9841,
"running": true,
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59004"
}
]
}
},
"lb": {
"id": "f7affe8e1b5904cb78f8ac5d43bab11da4006dd9d933118be1696ec36b8fc1b3",
"started_at": "2026-10-17T16:54:36.001647626Z",
"pid": 10960,
"running": true,
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9",
"memory": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8084"
}
]
}
}
},
"after": {
"postgres": {
"id": "a6e5f14597245ada8071635bf261c294e53045ee0d09bbea1cd682681cfacd9c",
"started_at": "2026-10-17T16:58:16.715706381Z",
"pid": 26327,
"running": true,
"image": "sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"5432/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "55435"
}
]
}
},
"valkey": {
"id": "1899ba79d40145cd3ff95269e73076e3b2ba74a472e4427f204da8e9c52592b6",
"started_at": "2026-10-17T16:58:16.815795964Z",
"pid": 26382,
"running": true,
"image": "sha256:d827e7f7552cdee40cc7482dbae9da020f42bc47669af6f71182a4ef76a22773",
"memory": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"6379/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "56382"
}
]
}
},
"s3": {
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
"started_at": "2026-10-17T16:58:16.681019131Z",
"pid": 26248,
"running": true,
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59004"
}
]
}
},
"lb": {
"id": "f7affe8e1b5904cb78f8ac5d43bab11da4006dd9d933118be1696ec36b8fc1b3",
"started_at": "2026-10-17T16:58:16.575992006Z",
"pid": 26158,
"running": true,
"image": "sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9",
"memory": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8084"
}
]
}
}
},
"same_token_public_test": true
},
"final_instances": [
{
"id": "ac20d4b45be85d82fe302e812b9b4c8d3645f68b126ef36b0aa1f0eae9f59a4d",
"started_at": "2026-10-17T16:58:16.863466756Z",
"pid": 26514,
"running": true,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "af0c8373068c4e0782fdb1af3602cac6"
},
{
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
"started_at": "2026-10-17T16:58:16.838496214Z",
"pid": 26426,
"running": true,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "e96c5c7e0457467597c53e1af497f8ac"
}
]
}
+4
View File
@@ -0,0 +1,4 @@
Verified failure and restoration of one API
Verified failure and restoration of one API
Verified sequential replacement under traffic
PASS: real replica failures, replay, rolling replacement and persistence
+3
View File
@@ -0,0 +1,3 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8084"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 3, "first_recovery_seconds": 0.769}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
uv run --locked ruff check src student-tests scripts/bootstrap.py scripts/cluster.py scripts/stop-one.py scripts/rolling-update.py scripts/verify_cluster.py scripts/verify_storage.py scripts/storage_lab.py research
All checks passed!
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8084"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+3
View File
@@ -0,0 +1,3 @@
Container mtusi-lab4-reconciler-run-de897097f2a4 Creating
Container mtusi-lab4-reconciler-run-de897097f2a4 Created
{"mode": "dry-run", "grace_seconds": 300, "elapsed_seconds": 0.10601412500000151, "counts": {"s3_list_pages": 1, "s3_objects": 50, "s3_head": 50, "delete_orphan": 2, "pg_metadata_pages": 2, "pg_attachments": 48, "ready_preserved": 48}, "actions": [{"action": "delete_orphan", "id": "2d392b8a-fbb1-444c-856e-f193ea279929"}, {"action": "delete_orphan", "id": "81c3c3e9-589f-4e4f-a130-c81082388565"}], "actions_preview_limit": 100}
+14
View File
@@ -0,0 +1,14 @@
{
"scope": "Редактура документации и метаданных рисунка",
"runtime_sources_unchanged": true,
"migrations_dependencies_and_infrastructure_unchanged": true,
"instructor_contract_and_public_tests_unchanged": true,
"measurement_values_unchanged": true,
"original_head": "da6ca232c350186445c8531a0fae52d657985569",
"rewritten_history_head": "36d295e16cc9477ce10dcf92aab2d80a6d3b826f",
"commit_map": {
"900182cbb405d57b20c1b8bfd7cef21592ccf4cf": "85433e20e4102202451b2358c4a65075b2964b27",
"da6ca232c350186445c8531a0fae52d657985569": "36d295e16cc9477ce10dcf92aab2d80a6d3b826f",
"ec540a1a725862a8619fcb758daec4ad197f0dd5": "ec540a1a725862a8619fcb758daec4ad197f0dd5"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"auth.py": "6bebcd8d1d3ca4c2c6e8b0e5793ae13d8996cac3709b5b845d814b51a5e1a3c3",
"service.py": "7e100756e3bb24a89bce5dfa41a471969cdb4287c1523673d1e7ff8255bf3c19",
"domain.py": "bd8456d7423a3e12c02a57ed90d1875fc5c7acdcfc0d26b94f9ead4855160298",
"cache.py": "17756fa84a2c40fd4d1baf9602a7b755ca5998bc241f0657db3505b04623321d",
"__init__.py": "f798dc149d6bef46d697dccdaf6be9c8d08654f18161e42c99f548c30074f4ae",
"attachments.py": "3fbf029536834be791d39b38fb76e2eb694b46a57d891ff73763b3a7be99234f",
"schemas.py": "3f94cc536beec31ec7890331abbc736445a311d03b30b9b5726dd60c17c9f6a9",
"storage.py": "2fd071caf423a09378d8fc162dc5f675484addf8589e0e0228057c6b7bb77d75",
"settings.py": "d21f4b624767b19c80f534e608de6de6071260b5e2d642a06e6a7aec0148c161",
"app.py": "6dd900a5ff7b2213ace024b55dfd7313195d1fb1257bab6067b16d2b4f515941",
"repository.py": "d29dabc547e0071b21f8b9d5371fc5753b89766f2c3e8ea8412cdcb8a8c33384",
"reconcile.py": "6c96d72a8c49a03ff721029545a41c3587bc24be3272df4003bf6141dd04df9c",
"migrate.py": "f073c080e13635a01482539dbb474b9b4f49bbf5ff66f483dbda4fbe1d1eb57f",
"__main__.py": "486cd5bbb3f1a814de74357e938968d219070e845cf178761a7a23e87d003ea4",
"cursor.py": "a2208cba232ae0f67c4430d3e90c3962129c8fb3987ea853c624cdd2c73d7a1f"
}
+3
View File
@@ -0,0 +1,3 @@
/binary/minio: go1.26.7
path github.com/minio/minio
mod github.com/minio/minio v0.0.0-20251015172955-9e49d5e7a648 h1:6TdolSCLSs2nwm8i0PpWDqf9iX2Ty9WQK8wmr7dCnUM=
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://127.0.0.1:8084"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+249
View File
@@ -0,0 +1,249 @@
{
"privacy": {
"anonymous_get_status": 403,
"api_delete_denied": true,
"other_bucket_denied": true,
"attachment": "d8693ef0-f3bc-4206-988f-d1664afeb827"
},
"cross_replica": {
"uploader": {
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
"started_at": "2026-10-17T16:58:16.838496214Z",
"pid": 26426,
"running": true,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "e96c5c7e0457467597c53e1af497f8ac"
},
"after_stop": {
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
"started_at": "2026-10-17T16:58:16.838496214Z",
"pid": 0,
"running": false,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {}
},
"downloader_instance": "af0c8373068c4e0782fdb1af3602cac6",
"original_preserved": true
},
"interruption": {
"before_rows": 5,
"after_rows": 5,
"aborted_body_no_intent": true
},
"uploads": [
{
"clients": 10,
"bytes_each": 1081015,
"pixels": 360000,
"duration_seconds": 0.16621475000101782,
"successful_rps": 60.16313233295339,
"errors": 0,
"p50_ms": 114.14141599925642,
"p95_ms": 153.93212499839137,
"p99_ms": 153.93212499839137,
"before": {
"09da358ec38461f3ee21c4d1129697a9f1486323410ddde90b5716268ac9e07c": {
"current_bytes": 95498240,
"peak_bytes": 170278912
},
"c0356d1a469d326b73ef6a577338bffb3860ff6afcc646dd5bf1198a0c5c1792": {
"current_bytes": 96903168,
"peak_bytes": 165269504
}
},
"after": {
"09da358ec38461f3ee21c4d1129697a9f1486323410ddde90b5716268ac9e07c": {
"current_bytes": 102379520,
"peak_bytes": 170278912
},
"c0356d1a469d326b73ef6a577338bffb3860ff6afcc646dd5bf1198a0c5c1792": {
"current_bytes": 101384192,
"peak_bytes": 165269504
}
},
"responses": [
{
"duration_ms": 76.5632500006177,
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
"id": "27d3163c-060f-4918-bed4-ec714a233595"
},
{
"duration_ms": 95.78312500161701,
"instance": "28dadbd8074d47a88abc712e0c76b41e",
"id": "f7796135-d0d4-4486-8281-6947ddfa0dee"
},
{
"duration_ms": 114.24379100026272,
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
"id": "fa09990f-7d53-4ee9-be56-e8153bd77eae"
},
{
"duration_ms": 114.14141599925642,
"instance": "28dadbd8074d47a88abc712e0c76b41e",
"id": "ccdfddef-f5fb-4c40-8d88-7049d5812f12"
},
{
"duration_ms": 133.79766599973664,
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
"id": "d3623f75-6e59-4f46-a809-a95590c89238"
},
{
"duration_ms": 153.14370899977803,
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
"id": "efd8eaa0-c524-4d93-8c4f-4e212af9f524"
},
{
"duration_ms": 94.01983300085703,
"instance": "3198dd66596b404b8d3e3f2d9cab7d81",
"id": "6810a6bc-af11-4d08-a007-d49908ce64df"
},
{
"duration_ms": 153.93212499839137,
"instance": "28dadbd8074d47a88abc712e0c76b41e",
"id": "3ab6e0ac-cf3f-4bcb-80d0-faf404fcea3b"
},
{
"duration_ms": 133.07204200100387,
"instance": "28dadbd8074d47a88abc712e0c76b41e",
"id": "f12f4260-b0fa-49aa-b4bc-2d0c6a4539a0"
},
{
"duration_ms": 75.5845829989994,
"instance": "28dadbd8074d47a88abc712e0c76b41e",
"id": "fecbb057-308a-45ec-9bcf-95856be457d7"
}
]
},
{
"clients": 10,
"bytes_each": 8674572,
"pixels": 2890000,
"duration_seconds": 0.7018210000005638,
"successful_rps": 14.248647447129633,
"errors": 0,
"p50_ms": 485.1206250004907,
"p95_ms": 683.7534580008651,
"p99_ms": 683.7534580008651,
"before": {
"4e4fab2b66feb2efc1c03e276733a0cd367a1d71d4c01714da527e4cfaa0f3db": {
"current_bytes": 95817728,
"peak_bytes": 170561536
},
"3e96dd5071d136499279b012cab6415e2a1d16bccb0605ab0db81f4f0d89de18": {
"current_bytes": 95109120,
"peak_bytes": 167698432
}
},
"after": {
"4e4fab2b66feb2efc1c03e276733a0cd367a1d71d4c01714da527e4cfaa0f3db": {
"current_bytes": 112578560,
"peak_bytes": 170561536
},
"3e96dd5071d136499279b012cab6415e2a1d16bccb0605ab0db81f4f0d89de18": {
"current_bytes": 112189440,
"peak_bytes": 167698432
}
},
"responses": [
{
"duration_ms": 650.3965840001911,
"instance": "fded6353f47f48bda9a66715f6e739d8",
"id": "d771f44e-cbcf-41de-9d6f-b3e8dbe6ac7d"
},
{
"duration_ms": 607.8036670005531,
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
"id": "2bf2e458-909b-4aaa-b939-69ad893a2191"
},
{
"duration_ms": 453.6054160016647,
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
"id": "26ad1005-42cc-486b-bdc6-f0f9a959d224"
},
{
"duration_ms": 485.1206250004907,
"instance": "fded6353f47f48bda9a66715f6e739d8",
"id": "a077b398-0d5d-43b9-8c94-1e42f0c61591"
},
{
"duration_ms": 328.2669590007572,
"instance": "fded6353f47f48bda9a66715f6e739d8",
"id": "7d293d77-2c70-4382-b1cd-14d318c33dde"
},
{
"duration_ms": 528.5843330002535,
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
"id": "11795c80-f431-46de-b672-005870afb0e0"
},
{
"duration_ms": 561.6339999996853,
"instance": "fded6353f47f48bda9a66715f6e739d8",
"id": "7d6915eb-e40d-4742-8590-1d6c482fc6a3"
},
{
"duration_ms": 410.78812499836204,
"instance": "fded6353f47f48bda9a66715f6e739d8",
"id": "394c46d5-af43-49ea-a6b2-bec2e5405343"
},
{
"duration_ms": 379.54966700090154,
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
"id": "40e7d89a-ad66-4f98-a5f6-076c25df99b9"
},
{
"duration_ms": 683.7534580008651,
"instance": "42f0e3d590f34d929f4ad18f3c1c61de",
"id": "0994d2cb-53ee-45fe-a20d-66b165f528d1"
}
]
}
],
"outage": {
"before": {
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
"started_at": "2026-10-17T16:58:16.681019131Z",
"pid": 26248,
"running": true,
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59004"
}
]
}
},
"after": {
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
"started_at": "2026-10-17T16:59:16.975092214Z",
"pid": 31660,
"running": true,
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59004"
}
]
}
},
"failed_operations": 2,
"statuses": [
503,
503
],
"stop_start_and_ready_seconds": 1.7791777080001339,
"original_preserved": true
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 80 KiB

+1
View File
@@ -0,0 +1 @@
{"orphan": "originals/bda0b2eb-b103-46de-9ba4-e08ca5a7580f", "created_at": "2026-10-17T16:56:00.179048+00:00"}
+675
View File
@@ -0,0 +1,675 @@
{
"started_at": "2026-10-17T17:04:23.007911+00:00",
"privacy": {
"anonymous_get_status": 403,
"api_delete_denied": true,
"other_bucket_denied": true,
"attachment": "d8693ef0-f3bc-4206-988f-d1664afeb827",
"console_unreachable_from_api": true
},
"cross_replica": {
"uploader": {
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
"started_at": "2026-10-17T16:58:16.838496214Z",
"pid": 26426,
"running": true,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"instance": "e96c5c7e0457467597c53e1af497f8ac"
},
"after_stop": {
"id": "e49421e41e2ac4cbeaf4784d2ba9ce5b47053021dc804a59cf85b2c096fbf87d",
"started_at": "2026-10-17T16:58:16.838496214Z",
"pid": 0,
"running": false,
"image": "sha256:6c726c0f2282b19aca4a42d1dcc517121d9994a44a00e288a6869ec31ab34ba6",
"memory": 536870912,
"nano_cpus": 1000000000,
"published_ports": {}
},
"downloader_instance": "af0c8373068c4e0782fdb1af3602cac6",
"original_preserved": true
},
"interruption": {
"before_rows": 5,
"after_rows": 5,
"aborted_body_no_intent": true
},
"uploads": [
{
"clients": 10,
"bytes_each": 1081015,
"pixels": 360000,
"duration_seconds": 0.17220016600003873,
"successful_rps": 58.07195331041522,
"resource_window_seconds": 0.34307429199907347,
"mean_api_cpu_cores_including_probes": 0.5834712908204168,
"errors": 0,
"p50_ms": 121.59408299885399,
"p95_ms": 159.44758300065587,
"p99_ms": 159.44758300065587,
"before": {
"eca892cc879ee0560cb843d51bf7db300892d6f38979b58e76371900e7c2090d": {
"current_bytes": 95768576,
"peak_bytes": 167612416,
"cpu": {
"usage_usec": 2004031,
"user_usec": 1828766,
"system_usec": 175265,
"nice_usec": 0,
"nr_periods": 27,
"nr_throttled": 14,
"throttled_usec": 1004492,
"nr_bursts": 0,
"burst_usec": 0
}
},
"e81b5efc6326bfe31b6674579ee91131656809855038b9792c4c31a029ffbc43": {
"current_bytes": 95109120,
"peak_bytes": 170680320,
"cpu": {
"usage_usec": 1977149,
"user_usec": 1830583,
"system_usec": 146566,
"nice_usec": 0,
"nr_periods": 27,
"nr_throttled": 12,
"throttled_usec": 1187980,
"nr_bursts": 0,
"burst_usec": 0
}
}
},
"after": {
"eca892cc879ee0560cb843d51bf7db300892d6f38979b58e76371900e7c2090d": {
"current_bytes": 102014976,
"peak_bytes": 167612416,
"cpu": {
"usage_usec": 2102645,
"user_usec": 1895893,
"system_usec": 206751,
"nice_usec": 0,
"nr_periods": 30,
"nr_throttled": 14,
"throttled_usec": 1004492,
"nr_bursts": 0,
"burst_usec": 0
}
},
"e81b5efc6326bfe31b6674579ee91131656809855038b9792c4c31a029ffbc43": {
"current_bytes": 101982208,
"peak_bytes": 170680320,
"cpu": {
"usage_usec": 2078709,
"user_usec": 1898994,
"system_usec": 179715,
"nice_usec": 0,
"nr_periods": 29,
"nr_throttled": 12,
"throttled_usec": 1187980,
"nr_bursts": 0,
"burst_usec": 0
}
}
},
"responses": [
{
"duration_ms": 122.1009580003738,
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
"id": "d2b7a203-e6c1-42e7-a798-c567fc956307"
},
{
"duration_ms": 121.59408299885399,
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
"id": "32fdc39b-f71f-4247-9e46-e0ac5bf042af"
},
{
"duration_ms": 159.44758300065587,
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
"id": "70ce907e-474b-4e34-8e31-ce3734f5937c"
},
{
"duration_ms": 140.0553330004186,
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
"id": "596a2e41-a075-4b2b-8c64-0ec5b7a26401"
},
{
"duration_ms": 159.16204099994502,
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
"id": "ae920131-d7f1-4b00-9083-58987593923d"
},
{
"duration_ms": 100.46366599999601,
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
"id": "8e788d3e-233e-4aaf-8306-aacea6b5719e"
},
{
"duration_ms": 83.1419169990113,
"instance": "f0d3185b4c134e53a139ee8e4400c33c",
"id": "e45403aa-fe8e-4485-9bfb-d3ab799064e8"
},
{
"duration_ms": 102.0030420004332,
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
"id": "542233b0-d431-4b36-a13b-e96250bcca0d"
},
{
"duration_ms": 141.32162500027334,
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
"id": "15e59f0b-1cd2-4464-88ef-c4a54b22345c"
},
{
"duration_ms": 82.26699999977427,
"instance": "2fbb3237281346958f1f4d3aeaf64e57",
"id": "2677cb60-6b1a-4d8f-9f66-c9a48e54cb06"
}
]
},
{
"clients": 10,
"bytes_each": 8674572,
"pixels": 2890000,
"duration_seconds": 0.7398240410002472,
"successful_rps": 13.516727553865282,
"resource_window_seconds": 0.9066179999990709,
"mean_api_cpu_cores_including_probes": 0.9118062954859126,
"errors": 0,
"p50_ms": 500.3316669990454,
"p95_ms": 726.9201670005714,
"p99_ms": 726.9201670005714,
"before": {
"0be0a33b995189ff06406219e52ce8253d76669ea7a27ec00e91e82bf0b3ddd5": {
"current_bytes": 97603584,
"peak_bytes": 170450944,
"cpu": {
"usage_usec": 2428375,
"user_usec": 2214869,
"system_usec": 213505,
"nice_usec": 0,
"nr_periods": 32,
"nr_throttled": 19,
"throttled_usec": 1560996,
"nr_bursts": 0,
"burst_usec": 0
}
},
"be0ea333307f7c2fe6ec09b49a476beea1b0af0e9ded9d6520ce342172bf296d": {
"current_bytes": 95272960,
"peak_bytes": 171569152,
"cpu": {
"usage_usec": 2446614,
"user_usec": 2215367,
"system_usec": 231247,
"nice_usec": 0,
"nr_periods": 33,
"nr_throttled": 16,
"throttled_usec": 1407896,
"nr_bursts": 0,
"burst_usec": 0
}
}
},
"after": {
"0be0a33b995189ff06406219e52ce8253d76669ea7a27ec00e91e82bf0b3ddd5": {
"current_bytes": 110301184,
"peak_bytes": 170450944,
"cpu": {
"usage_usec": 2855656,
"user_usec": 2474167,
"system_usec": 381488,
"nice_usec": 0,
"nr_periods": 40,
"nr_throttled": 19,
"throttled_usec": 1560996,
"nr_bursts": 0,
"burst_usec": 0
}
},
"be0ea333307f7c2fe6ec09b49a476beea1b0af0e9ded9d6520ce342172bf296d": {
"current_bytes": 111939584,
"peak_bytes": 171569152,
"cpu": {
"usage_usec": 2845993,
"user_usec": 2465191,
"system_usec": 380801,
"nice_usec": 0,
"nr_periods": 41,
"nr_throttled": 16,
"throttled_usec": 1407896,
"nr_bursts": 0,
"burst_usec": 0
}
}
},
"responses": [
{
"duration_ms": 437.1919159984827,
"instance": "40f46dd47d7f4b539d5d50346140bb29",
"id": "53f9fa9d-05e3-4f31-9b88-0d721cd86844"
},
{
"duration_ms": 500.3316669990454,
"instance": "40f46dd47d7f4b539d5d50346140bb29",
"id": "00d1c78e-0639-4430-9bec-624c7a458f35"
},
{
"duration_ms": 639.1502089991263,
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
"id": "459b42a4-d28d-40ce-a34c-71d49d2d0dda"
},
{
"duration_ms": 479.9554580004042,
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
"id": "2fd30bb2-9436-4ac0-a962-d3dcbf699aee"
},
{
"duration_ms": 561.1379999991186,
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
"id": "49680954-5763-44f0-8073-0ece17bbf125"
},
{
"duration_ms": 593.6962920004589,
"instance": "40f46dd47d7f4b539d5d50346140bb29",
"id": "d79bb81d-be37-4146-a347-7d3e47c1c429"
},
{
"duration_ms": 672.7730829989014,
"instance": "40f46dd47d7f4b539d5d50346140bb29",
"id": "719e4162-0ac2-463b-b9bf-fec4b3ed2015"
},
{
"duration_ms": 726.9201670005714,
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
"id": "bb116ad8-87c1-4fcc-863b-399cf0f8f2fd"
},
{
"duration_ms": 317.0152499988035,
"instance": "40f46dd47d7f4b539d5d50346140bb29",
"id": "db210a76-b08b-492e-a552-d1d33107569b"
},
{
"duration_ms": 401.8378750006377,
"instance": "80b2d2f42a4d451da0a5236b599c9d92",
"id": "3323d70f-f307-4d16-9635-fab78ae143f5"
}
]
}
],
"outage": {
"before": {
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
"started_at": "2026-10-17T16:58:16.681019131Z",
"pid": 26248,
"running": true,
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59004"
}
]
}
},
"after": {
"id": "69b2b1d137ebb6ad0c9790189e0d858267596ea1481297fe1db1d21b37a25002",
"started_at": "2026-10-17T16:59:16.975092214Z",
"pid": 31660,
"running": true,
"image": "sha256:bd904439d1bb8ba4bbff8886ca4b91eb446b01891cc4ee8a4668a38b700bad7e",
"memory": 805306368,
"nano_cpus": 1000000000,
"published_ports": {
"9000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "59004"
}
]
}
},
"failed_operations": 2,
"statuses": [
503,
503
],
"stop_start_and_ready_seconds": 1.7791777080001339,
"original_preserved": true
},
"reconciliation": {
"old_orphan_age_seconds": 503.0134530067444,
"dry_run": {
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.18648320799911744,
"counts": {
"s3_list_pages": 1,
"s3_objects": 31,
"fresh_objects_skipped": 4,
"s3_head": 30,
"delete_orphan": 1,
"pg_metadata_pages": 2,
"pg_attachments": 29,
"ready_preserved": 28,
"delete_abandoned_intent": 1
},
"actions": [
{
"action": "delete_orphan",
"id": "bda0b2eb-b103-46de-9ba4-e08ca5a7580f"
},
{
"action": "delete_abandoned_intent",
"id": "f878fa3a-617d-476c-90d5-1b94404367b9"
}
],
"actions_preview_limit": 100
},
"apply": {
"mode": "apply",
"grace_seconds": 300,
"elapsed_seconds": 0.13833720799993898,
"counts": {
"s3_list_pages": 1,
"s3_objects": 31,
"fresh_objects_skipped": 4,
"s3_head": 30,
"delete_orphan": 1,
"s3_delete": 1,
"pg_metadata_pages": 2,
"pg_attachments": 29,
"ready_preserved": 28,
"delete_abandoned_intent": 1
},
"actions": [
{
"action": "delete_orphan",
"id": "bda0b2eb-b103-46de-9ba4-e08ca5a7580f"
},
{
"action": "delete_abandoned_intent",
"id": "f878fa3a-617d-476c-90d5-1b94404367b9"
}
],
"actions_preview_limit": 100
},
"repeat": {
"mode": "apply",
"grace_seconds": 300,
"elapsed_seconds": 0.12051066699859803,
"counts": {
"s3_list_pages": 1,
"s3_objects": 30,
"fresh_objects_skipped": 4,
"pg_metadata_pages": 2,
"pg_attachments": 28,
"s3_head": 28,
"ready_preserved": 28
},
"actions": [],
"actions_preview_limit": 100
},
"missing_dry_run": {
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.11716975000126695,
"counts": {
"s3_list_pages": 1,
"s3_objects": 29,
"fresh_objects_skipped": 3,
"pg_metadata_pages": 2,
"pg_attachments": 28,
"s3_head": 28,
"ready_preserved": 27,
"mark_missing": 1
},
"actions": [
{
"action": "mark_missing",
"id": "b3cb28f8-4ec0-4467-8d62-4ddce58c5888"
}
],
"actions_preview_limit": 100
},
"missing_apply": {
"mode": "apply",
"grace_seconds": 300,
"elapsed_seconds": 0.12134083299861231,
"counts": {
"s3_list_pages": 1,
"s3_objects": 29,
"fresh_objects_skipped": 3,
"pg_metadata_pages": 2,
"pg_attachments": 28,
"s3_head": 28,
"ready_preserved": 27,
"mark_missing": 1
},
"actions": [
{
"action": "mark_missing",
"id": "b3cb28f8-4ec0-4467-8d62-4ddce58c5888"
}
],
"actions_preview_limit": 100
},
"healing": {
"mode": "apply",
"grace_seconds": 300,
"elapsed_seconds": 0.11887154199939687,
"counts": {
"s3_list_pages": 1,
"s3_objects": 30,
"fresh_objects_skipped": 4,
"pg_metadata_pages": 2,
"pg_attachments": 28,
"s3_head": 28,
"ready_preserved": 27,
"restore_ready": 1
},
"actions": [
{
"action": "restore_ready",
"id": "b3cb28f8-4ec0-4467-8d62-4ddce58c5888"
}
],
"actions_preview_limit": 100
},
"fresh_preserved": true,
"used_metadata_and_link_preserved": true
},
"scan_cost": [
{
"extra_fresh_objects": 0,
"repeats": [
{
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.18440870900121809,
"counts": {
"s3_list_pages": 1,
"s3_objects": 50,
"fresh_objects_skipped": 24,
"pg_metadata_pages": 2,
"pg_attachments": 48,
"fresh_metadata_skipped": 21,
"s3_head": 27,
"ready_preserved": 27
},
"actions": [],
"actions_preview_limit": 100
},
{
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.14119874999960302,
"counts": {
"s3_list_pages": 1,
"s3_objects": 50,
"fresh_objects_skipped": 24,
"pg_metadata_pages": 2,
"pg_attachments": 48,
"fresh_metadata_skipped": 21,
"s3_head": 27,
"ready_preserved": 27
},
"actions": [],
"actions_preview_limit": 100
},
{
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.13778470800025389,
"counts": {
"s3_list_pages": 1,
"s3_objects": 50,
"fresh_objects_skipped": 24,
"pg_metadata_pages": 2,
"pg_attachments": 48,
"fresh_metadata_skipped": 21,
"s3_head": 27,
"ready_preserved": 27
},
"actions": [],
"actions_preview_limit": 100
}
]
},
{
"extra_fresh_objects": 100,
"repeats": [
{
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.16300708299968392,
"counts": {
"s3_list_pages": 1,
"s3_objects": 150,
"fresh_objects_skipped": 124,
"pg_metadata_pages": 2,
"pg_attachments": 48,
"fresh_metadata_skipped": 21,
"s3_head": 27,
"ready_preserved": 27
},
"actions": [],
"actions_preview_limit": 100
},
{
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.15059724999991886,
"counts": {
"s3_list_pages": 1,
"s3_objects": 150,
"fresh_objects_skipped": 124,
"pg_metadata_pages": 2,
"pg_attachments": 48,
"fresh_metadata_skipped": 21,
"s3_head": 27,
"ready_preserved": 27
},
"actions": [],
"actions_preview_limit": 100
},
{
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.16615074999936041,
"counts": {
"s3_list_pages": 1,
"s3_objects": 150,
"fresh_objects_skipped": 124,
"pg_metadata_pages": 2,
"pg_attachments": 48,
"fresh_metadata_skipped": 21,
"s3_head": 27,
"ready_preserved": 27
},
"actions": [],
"actions_preview_limit": 100
}
]
},
{
"extra_fresh_objects": 1100,
"repeats": [
{
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.21618029099954583,
"counts": {
"s3_list_pages": 2,
"s3_objects": 1150,
"fresh_objects_skipped": 1124,
"pg_metadata_pages": 2,
"pg_attachments": 48,
"fresh_metadata_skipped": 21,
"s3_head": 27,
"ready_preserved": 27
},
"actions": [],
"actions_preview_limit": 100
},
{
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.23353216699979384,
"counts": {
"s3_list_pages": 2,
"s3_objects": 1150,
"fresh_objects_skipped": 1124,
"pg_metadata_pages": 2,
"pg_attachments": 48,
"fresh_metadata_skipped": 21,
"s3_head": 27,
"ready_preserved": 27
},
"actions": [],
"actions_preview_limit": 100
},
{
"mode": "dry-run",
"grace_seconds": 300,
"elapsed_seconds": 0.18981450000137556,
"counts": {
"s3_list_pages": 2,
"s3_objects": 1150,
"fresh_objects_skipped": 1124,
"pg_metadata_pages": 2,
"pg_attachments": 48,
"fresh_metadata_skipped": 21,
"s3_head": 27,
"ready_preserved": 27
},
"actions": [],
"actions_preview_limit": 100
}
]
}
],
"finished_at": "2026-10-17T17:04:32.396891+00:00",
"measurement_runtime_sha256": {
"reconcile.py": "6c96d72a8c49a03ff721029545a41c3587bc24be3272df4003bf6141dd04df9c",
"cursor.py": "a2208cba232ae0f67c4430d3e90c3962129c8fb3987ea853c624cdd2c73d7a1f",
"cache.py": "17756fa84a2c40fd4d1baf9602a7b755ca5998bc241f0657db3505b04623321d",
"service.py": "7e100756e3bb24a89bce5dfa41a471969cdb4287c1523673d1e7ff8255bf3c19",
"__init__.py": "f798dc149d6bef46d697dccdaf6be9c8d08654f18161e42c99f548c30074f4ae",
"__main__.py": "486cd5bbb3f1a814de74357e938968d219070e845cf178761a7a23e87d003ea4",
"app.py": "6dd900a5ff7b2213ace024b55dfd7313195d1fb1257bab6067b16d2b4f515941",
"settings.py": "d21f4b624767b19c80f534e608de6de6071260b5e2d642a06e6a7aec0148c161",
"auth.py": "6bebcd8d1d3ca4c2c6e8b0e5793ae13d8996cac3709b5b845d814b51a5e1a3c3",
"attachments.py": "3fbf029536834be791d39b38fb76e2eb694b46a57d891ff73763b3a7be99234f",
"storage.py": "2fd071caf423a09378d8fc162dc5f675484addf8589e0e0228057c6b7bb77d75",
"migrate.py": "f073c080e13635a01482539dbb474b9b4f49bbf5ff66f483dbda4fbe1d1eb57f",
"schemas.py": "3f94cc536beec31ec7890331abbc736445a311d03b30b9b5726dd60c17c9f6a9",
"repository.py": "d29dabc547e0071b21f8b9d5371fc5753b89766f2c3e8ea8412cdcb8a8c33384",
"domain.py": "bd8456d7423a3e12c02a57ed90d1875fc5c7acdcfc0d26b94f9ead4855160298"
},
"measurements_after_checks_completed": true,
"bucket_bootstrap_repeated": 2
}
+1
View File
@@ -0,0 +1 @@
PASS: private S3, upload memory bound, interruption, outage and reconciler
+13
View File
@@ -0,0 +1,13 @@
============================= test session starts ==============================
platform darwin -- Python 3.14.7, pytest-9.1.1, pluggy-1.7.0
rootdir: <repository>
configfile: pyproject.toml
testpaths: student-tests
plugins: anyio-4.15.1
collected 65 items
student-tests/test_api.py .................................... [ 55%]
student-tests/test_attachments.py ................. [ 81%]
student-tests/test_cluster.py ............ [100%]
======================= 65 passed, 2 warnings in 15.34s ========================
+90
View File
@@ -0,0 +1,90 @@
{
"project": "mtusi-lab4",
"services": {
"api": {
"image": "mtusi-messenger:lab4",
"cpus": 1,
"mem_limit": "536870912",
"memswap_limit": "536870912",
"user": "10001:10001",
"read_only": true,
"secrets": [
"database_url",
"cache_url",
"s3_api_access_key_id",
"s3_api_secret_access_key"
]
},
"bucket-init": {
"image": "mtusi-lab4-mc:2025-08-13",
"cpus": 0.5,
"mem_limit": "134217728",
"read_only": true,
"secrets": [
"s3_root_user",
"s3_root_password",
"s3_api_access_key_id",
"s3_api_secret_access_key",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
"s3_test_access_key_id",
"s3_test_secret_access_key"
]
},
"lb": {
"image": "haproxy:3.2.1-alpine",
"cpus": 0.5,
"mem_limit": "134217728",
"memswap_limit": "134217728",
"ports": [
{
"mode": "ingress",
"host_ip": "127.0.0.1",
"target": 8080,
"published": "8084",
"protocol": "tcp"
}
],
"read_only": true,
"secrets": []
},
"migrate": {
"image": "mtusi-messenger:lab4",
"cpus": 1,
"mem_limit": "536870912",
"memswap_limit": "536870912",
"user": "10001:10001",
"read_only": true,
"secrets": [
"database_url"
]
},
"postgres": {
"image": "postgres:18.3",
"cpus": 1,
"mem_limit": "805306368",
"memswap_limit": "805306368",
"secrets": [
"postgres_password"
]
},
"s3": {
"image": "mtusi-lab4-s3:2025-10-15",
"cpus": 1,
"mem_limit": "805306368",
"memswap_limit": "805306368",
"read_only": true,
"secrets": [
"s3_root_user",
"s3_root_password"
]
},
"valkey": {
"image": "valkey/valkey:8.1.3-alpine",
"cpus": 0.5,
"mem_limit": "134217728",
"memswap_limit": "134217728",
"secrets": []
}
}
}
+8
View File
@@ -0,0 +1,8 @@
Python 3.14.7
uv 0.12.13 (Homebrew 2026-09-10 aarch64-apple-darwin)
Docker client 29.8.1; server 29.8.1
5.5.1
minio version DEVELOPMENT.GOGET (commit-id=DEVELOPMENT.GOGET)
Runtime: go1.26.7 linux/arm64
License: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html
Copyright: 2015-0000 MinIO, Inc.
+1
View File
@@ -0,0 +1 @@
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::BUCKET"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject"],"Resource":["arn:aws:s3:::BUCKET/originals/*"]}]}
+29
View File
@@ -0,0 +1,29 @@
#!/bin/sh
set -eu
# Never print mc arguments/errors: they can include access key identifiers.
exec 3>&1
exec > /tmp/bootstrap.log 2>&1
trap 'printf "%s\n" "S3 bootstrap failed; credentials suppressed" >&3' EXIT
bucket="$S3_BUCKET"
case "$bucket" in ''|*[!a-z0-9.-]*) exit 2;; esac
mc --config-dir /tmp/mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)"
mc --config-dir /tmp/mc mb --ignore-existing "local/$bucket"
mc --config-dir /tmp/mc anonymous set none "local/$bucket"
for role in api reconciler test; do
target_bucket="$bucket"
policy_source=/policies/api-policy.json
if [ "$role" = reconciler ]; then policy_source=/policies/reconciler-policy.json; fi
if [ "$role" = test ]; then
if [ "${CREATE_TEST_BUCKET:-0}" != 1 ]; then continue; fi
target_bucket="$bucket-tests"
mc --config-dir /tmp/mc mb --ignore-existing "local/$target_bucket"
mc --config-dir /tmp/mc anonymous set none "local/$target_bucket"
fi
sed "s/BUCKET/$target_bucket/g" "$policy_source" > "/tmp/$role-policy.json"
mc --config-dir /tmp/mc admin policy create local "messenger-$role" "/tmp/$role-policy.json"
key="$(cat /run/secrets/s3_${role}_access_key_id)"
mc --config-dir /tmp/mc admin user add local "$key" "$(cat /run/secrets/s3_${role}_secret_access_key)"
mc --config-dir /tmp/mc admin policy attach local "messenger-$role" --user "$key"
done
trap - EXIT
printf '%s\n' 'PASS: private bucket and scoped identities ready' >&3
+44
View File
@@ -0,0 +1,44 @@
global
log stdout format raw local0
maxconn 2048
defaults
log global
mode http
log-format "lb status=%ST backend=%b server=%s duration_ms=%Tt"
timeout connect 500ms
timeout check 500ms
timeout client 20s
timeout server 30s
timeout http-request 3s
retries 0
resolvers docker
nameserver embedded 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 0s
hold nx 1s
hold refused 1s
hold timeout 1s
hold other 1s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 1
server-template app 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
listen metrics
bind :8404
stats enable
stats uri /stats
stats refresh 1s
+1
View File
@@ -0,0 +1 @@
{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::BUCKET"]},{"Effect":"Allow","Action":["s3:GetObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::BUCKET/originals/*"]}]}
+18
View File
@@ -0,0 +1,18 @@
# Official security-fixed source release; no dependency on retired image tags.
FROM golang:1.26.7-alpine3.23@sha256:b17af760035fc2f338eed92d448a6c67f2d45438844fc6c60678fa5f99e44b57 AS builder
ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOMAXPROCS=4 GOFLAGS=-p=4
RUN go install -ldflags='-s -w' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
RUN go install -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS server
COPY --from=builder /go/bin/minio /usr/local/bin/minio
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
RUN addgroup -g 10001 app && adduser -D -u 10001 -G app app && mkdir /data && chown app:app /data
COPY --chmod=755 infra/start-s3.sh /usr/local/bin/start-s3.sh
USER 10001:10001
ENTRYPOINT ["/usr/local/bin/start-s3.sh"]
FROM alpine:3.23@sha256:85fe1e81d6758c208f3e1eed4338a1997e19d4be002d4dd32d3100c9a8c010a0 AS client
COPY --from=builder /go/bin/mc /usr/local/bin/mc
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
ENTRYPOINT ["/bin/sh"]
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)"
exec minio "$@"
+37
View File
@@ -0,0 +1,37 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username varchar(32) COLLATE "C" NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
PRIMARY KEY (chat_id,user_id),
UNIQUE (chat_id,position)
);
CREATE INDEX chat_members_by_user ON chat_members (user_id,chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX messages_history ON messages (chat_id,created_at,id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash)=32),
user_id uuid NOT NULL REFERENCES users(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry ON sessions (expires_at);
+14
View File
@@ -0,0 +1,14 @@
CREATE TABLE message_requests (
user_id uuid NOT NULL REFERENCES users(id),
http_method text NOT NULL DEFAULT 'POST' CHECK (http_method='POST'),
chat_id uuid NOT NULL REFERENCES chats(id),
idempotency_key varchar(128) COLLATE "C" NOT NULL
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
payload_hash bytea NOT NULL CHECK (octet_length(payload_hash)=32),
response_message_id uuid NOT NULL REFERENCES messages(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id,http_method,chat_id,idempotency_key),
CHECK (expires_at > created_at)
);
CREATE INDEX message_requests_expiry ON message_requests (expires_at);
+30
View File
@@ -0,0 +1,30 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
storage_state text NOT NULL CHECK (storage_state IN ('pending','ready','missing')),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE (id,chat_id),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX attachments_reconcile ON attachments (updated_at,id);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CHECK (char_length(text) BETWEEN 0 AND 4000);
ALTER TABLE messages ADD UNIQUE (id,chat_id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id,attachment_id),
UNIQUE (message_id,position),
FOREIGN KEY (message_id,chat_id) REFERENCES messages(id,chat_id),
FOREIGN KEY (attachment_id,chat_id) REFERENCES attachments(id,chat_id)
);
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
+45
View File
@@ -0,0 +1,45 @@
[project]
name = "mtusi-messenger"
version = "0.4.0"
description = "MTUSI lab 4: private object storage and recoverable attachments"
requires-python = ">=3.14,<3.15"
dependencies = [
"argon2-cffi==25.1.0",
"boto3==1.43.109",
"fastapi==0.143.3",
"pillow==13.0.0",
"psycopg[binary,pool]==3.3.6",
"pydantic==2.14.1",
"redis==8.1.0",
"uvicorn==0.54.0",
]
[build-system]
requires = ["hatchling==1.27.0"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/messenger"]
[dependency-groups]
dev = [
"httpx==0.28.1",
"openapi-spec-validator==0.7.2",
"pytest==9.1.1",
"ruff==0.17.1",
]
research = ["matplotlib==3.11.2"]
[tool.pytest.ini_options]
testpaths = ["student-tests"]
addopts = "-ra --tb=short"
[tool.ruff]
target-version = "py314"
line-length = 100
[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
[tool.ruff.per-file-target-version]
"scripts/*.py" = "py39"
+58
View File
@@ -0,0 +1,58 @@
"""Render measured kernel memory peaks and dry-run traversal costs."""
import argparse
import json
from pathlib import Path
from statistics import median
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("input", nargs="?", type=Path, default=ROOT / "evidence/storage.json")
args = parser.parse_args()
d = json.loads(args.input.read_text())
plt.rcParams.update(
{
"font.size": 10,
"figure.dpi": 150,
"axes.grid": True,
"grid.alpha": 0.2,
"axes.spines.top": False,
"axes.spines.right": False,
}
)
fig, ax = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True)
labels = [f"{r['bytes_each'] / 1048576:.2f} MiB" for r in d["uploads"]]
base = [max(v["current_bytes"] for v in r["before"].values()) / 1048576 for r in d["uploads"]]
peak = [max(v["peak_bytes"] for v in r["after"].values()) / 1048576 for r in d["uploads"]]
x = list(range(len(labels)))
ax[0].bar([v - 0.18 for v in x], base, width=0.36, label="Before: max API memory.current")
ax[0].bar([v + 0.18 for v in x], peak, width=0.36, label="Kernel memory.peak incl. startup")
ax[0].axhline(512, color="gray", linestyle="--", label="Per API limit: 512 MiB")
ax[0].set(
xticks=x,
xticklabels=labels,
ylabel="MiB per API container",
title="10 simultaneous clients, 1 active upload/API",
)
ax[0].legend(fontsize=7)
cost = d["scan_cost"]
xs = [r["extra_fresh_objects"] for r in cost]
values = [[v["elapsed_seconds"] * 1000 for v in r["repeats"]] for r in cost]
ax[1].plot(xs, [median(v) for v in values], marker="o", label="Median of 3 dry runs")
ax[1].fill_between(
xs, [min(v) for v in values], [max(v) for v in values], alpha=0.15, label="Min/max"
)
ax[1].set(
xlabel="Extra fresh objects (metadata unchanged)",
ylabel="Traversal wall time, ms",
title="LIST pagination + metadata HEAD checks",
)
ax[1].legend(fontsize=8)
fig.savefig(args.input.with_name(args.input.stem + "-research.png"))
plt.close(fig)
print("PASS: research figure generated from storage.json")
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Create local synthetic credentials once, without printing their values."""
import os
import secrets
from pathlib import Path
root = Path(__file__).resolve().parents[1]
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
password_file = directory / "postgres_password"
if not password_file.exists():
password_file.write_text(secrets.token_urlsafe(32) + "\n")
password = password_file.read_text().strip()
for name, value in {
"database_url": f"postgresql://messenger:{password}@postgres:5432/messenger",
"cache_url": "redis://valkey:6379/0",
}.items():
path = directory / name
if not path.exists():
path.write_text(value + "\n")
# Compose bind-mount secrets must be readable by the non-root container process.
# The parent directory remains 0700 on the host.
for role in ("root", "api", "reconciler", "test"):
names = (
("s3_root_user", "s3_root_password")
if role == "root"
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
)
for index, name in enumerate(names):
path = directory / name
if not path.exists():
path.write_text(
(secrets.token_hex(10) if index == 0 else secrets.token_urlsafe(32)) + "\n"
)
for name in (
"postgres_password",
"database_url",
"cache_url",
*[p.name for p in directory.glob("s3_*")],
):
os.chmod(directory / name, 0o644)
env = root / ".env"
if not env.exists():
env.write_text((root / ".env.example").read_text())
print("Local secret files and environment are ready; existing values preserved.")
+82
View File
@@ -0,0 +1,82 @@
"""Read-only discovery and scoped Compose operations for the local lab4 stack."""
import json
import subprocess
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(*args, env=None):
return subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
timeout=90,
).stdout
def inspect(container):
template = (
'{"id":{{json .Id}},"started_at":{{json .State.StartedAt}},'
'"pid":{{.State.Pid}},"running":{{json .State.Running}},'
'"image":{{json .Image}},"memory":{{.HostConfig.Memory}},'
'"nano_cpus":{{.HostConfig.NanoCpus}},'
'"published_ports":{{json .HostConfig.PortBindings}}}'
)
r = subprocess.run(
["docker", "inspect", "--format", template, container],
capture_output=True,
text=True,
check=True,
timeout=5,
)
return json.loads(r.stdout)
def instance(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
capture_output=True,
text=True,
check=True,
timeout=3,
)
value = inspect(container)
value["instance"] = r.stdout.strip()
return value
def instances():
ids = compose("ps", "-q", "api").split()
with ThreadPoolExecutor(max_workers=8) as pool:
return list(pool.map(instance, ids))
def wait_lb(base="http://127.0.0.1:8084", count=2, timeout=45):
deadline, seen = time.monotonic() + timeout, set()
while time.monotonic() < deadline:
try:
current = {value["instance"] for value in instances()}
if len(current) != count:
time.sleep(0.2)
continue
with urllib.request.urlopen(base + "/health/ready", timeout=2) as response:
seen.add(response.headers["X-Instance-Id"])
if current <= seen:
return sorted(current)
except (OSError, urllib.error.URLError, subprocess.SubprocessError):
pass
time.sleep(0.1)
raise RuntimeError("Expected healthy API replicas were not observed through LB")
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# All running application/state services; no migration, down -v, flush or seed.
docker compose --env-file .env restart --timeout 7 api postgres valkey s3 lb
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Sequential process replacement; at least one existing API stays running."""
import argparse
import json
import subprocess
from cluster import compose, instances, wait_lb
def replace_all(base="http://127.0.0.1:8084"):
original = instances()
if len(original) < 2:
raise RuntimeError("Rolling replacement requires at least two healthy replicas")
wait_lb(base, len(original))
replacements = []
for current in original:
subprocess.run(
["docker", "stop", "--time", "5", current["id"]],
check=True,
capture_output=True,
timeout=8,
)
subprocess.run(["docker", "rm", current["id"]], check=True, capture_output=True)
compose(
"up", "-d", "--no-deps", "--no-recreate", "--scale", "api=" + str(len(original)), "api"
)
aliases = wait_lb(base, len(original))
if current["instance"] in aliases:
raise RuntimeError("Old API process is still serving after replacement")
replacements.append(
{
"old_container": current["id"],
"old_instance": current["instance"],
"live_instances": aliases,
}
)
return replacements
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
args = parser.parse_args()
print(json.dumps(replace_all(args.base_url)))
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""Map a process alias to ONE container belonging to this Compose service."""
import json
import os
import re
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def alias(container):
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live',timeout=1); print(r.headers['X-Instance-Id'])"
)
r = subprocess.run(
["docker", "exec", container, "python", "-c", code],
cwd=ROOT,
capture_output=True,
text=True,
timeout=3,
check=True,
)
return container, r.stdout.strip()
def main():
target = os.environ.get("TARGET_INSTANCE_ID", "")
if not re.fullmatch(r"[a-f0-9]{32}", target):
raise SystemExit("TARGET_INSTANCE_ID must name a live API process")
ids = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
with ThreadPoolExecutor(max_workers=8) as pool:
matches = [cid for cid, identity in pool.map(alias, ids) if identity == target]
if len(matches) != 1:
raise SystemExit("Expected exactly one matching API in this project")
subprocess.run(
["docker", "stop", "--time", "5", matches[0]], check=True, capture_output=True, timeout=7
)
print(json.dumps({"stopped_container": matches[0], "instance": target}))
if __name__ == "__main__":
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/stop-one.py
+45
View File
@@ -0,0 +1,45 @@
"""Local-only operator helpers; secret values never leave this process."""
from pathlib import Path
import psycopg
from psycopg.conninfo import make_conninfo
from psycopg.rows import dict_row
from messenger.settings import Settings
from messenger.storage import ObjectStore
ROOT = Path(__file__).resolve().parents[1]
def settings(role="reconciler"):
password = (ROOT / "secrets/postgres_password").read_text().strip()
dsn = make_conninfo(
host="127.0.0.1",
port=55435,
dbname="messenger",
user="messenger",
password=password,
connect_timeout=2,
options="-c timezone=UTC",
)
key, secret = (
("s3_root_user", "s3_root_password")
if role == "root"
else ("s3_" + role + "_access_key_id", "s3_" + role + "_secret_access_key")
)
return Settings(
dsn,
"redis://127.0.0.1:56382/0",
s3_access_key=(ROOT / "secrets" / key).read_text().strip(),
s3_secret_key=(ROOT / "secrets" / secret).read_text().strip(),
)
def connection(config=None):
config = config or settings()
return psycopg.connect(config.database_url, autocommit=True, row_factory=dict_row)
def store(role="reconciler"):
return ObjectStore(settings(role))
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env python3
"""Real LB requests during each replica failure and sequential replacement."""
import argparse
import json
import os
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from uuid import uuid4
from cluster import ROOT, compose, inspect, instance, instances, wait_lb
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from resilience import history # noqa: E402
def percentile(values, fraction):
values = sorted(values)
return values[min(int((len(values) - 1) * fraction), len(values) - 1)] if values else None
class Traffic:
def __init__(self, base, actor, room, workers=6):
self.base, self.actor, self.room = base, actor, room
self.prefix = uuid4().hex
self.stop = threading.Event()
self.lock = threading.Lock()
self.events = []
self.executor = ThreadPoolExecutor(max_workers=workers)
self.workers = workers
def worker(self, worker):
client, iteration = Client(self.base, timeout=3), 0
while not self.stop.is_set():
started = time.monotonic()
write = iteration % 10 == 9
try:
if write:
key = f"{self.prefix}:{worker}:{iteration}"
response = client.message(self.actor, self.room, key, key=key)
else:
response = client.call(
"GET", "/api/v1/users/me", actor=self.actor, schema="User"
)
event = {
"at": time.monotonic(),
"ok": True,
"instance": response["headers"]["X-Instance-Id"],
"latency_ms": (time.monotonic() - started) * 1000,
"message": response["json"] if write else None,
}
iteration += 1
except Failure:
event = {
"at": time.monotonic(),
"ok": False,
"latency_ms": (time.monotonic() - started) * 1000,
}
with self.lock:
self.events.append(event)
def __enter__(self):
self.started = time.monotonic()
self.futures = [self.executor.submit(self.worker, i) for i in range(self.workers)]
return self
def __exit__(self, *args):
self.stop.set()
self.executor.shutdown(wait=True)
for future in self.futures:
future.result()
self.finished = time.monotonic()
def report(self, action_start, action_end):
events = sorted(self.events, key=lambda row: row["at"])
ok = [row for row in events if row["ok"]]
after = [row for row in ok if row["at"] >= action_end]
check(bool(after), "No successful request after action")
elapsed = self.finished - self.started
latency = [row["latency_ms"] for row in ok]
return {
"duration_seconds": elapsed,
"workers": self.workers,
"requests": len(events),
"successes": len(ok),
"errors": len(events) - len(ok),
"successful_rps": len(ok) / elapsed,
"error_fraction": (len(events) - len(ok)) / len(events),
"first_success_after_action_seconds": after[0]["at"] - action_start,
"action_seconds": action_end - action_start,
"p50_ms": percentile(latency, 0.5),
"p95_ms": percentile(latency, 0.95),
"p99_ms": percentile(latency, 0.99),
"instances": sorted({row["instance"] for row in ok}),
"tail_instances": sorted({row["instance"] for row in after[-10:]}),
"confirmed_writes": sum(row["message"] is not None for row in ok),
}
def verify_messages(self, client, actor, room):
actual = history(client, actor, room)
by_id = {value["id"]: value for value in actual}
check(len(by_id) == len(actual), "History has duplicate IDs")
texts = [row["text"] for row in actual if row["text"].startswith(self.prefix)]
check(len(texts) == len(set(texts)), "An uncertain retry created a duplicate effect")
for event in self.events:
if event["ok"] and event["message"]:
check(
by_id.get(event["message"]["id"]) == event["message"],
"A confirmed message was lost or changed",
)
def public_test(target, base):
result = subprocess.run(
[
"make",
target,
"ACTION_SCRIPT=scripts/"
+ ("restart.sh" if target == "test-persistence" else "stop-one.sh"),
"BASE_URL=" + base,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=150,
)
return "\n".join(line.rstrip() for line in result.stdout.splitlines()) + "\n"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
parser.add_argument(
"--output", type=Path, default=ROOT / f"evidence/cluster-{time.time_ns()}.json"
)
args = parser.parse_args()
if args.output.exists():
raise SystemExit("Use a new evidence filename")
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "127.0.0.1,localhost"
baseline = instances()
check(len(baseline) == 2, "This verification expects the default two API replicas")
check(not any(row["published_ports"] for row in baseline), "API must not publish host ports")
wait_lb(args.base_url, 2)
result = {"initial_instances": baseline, "failures": []}
client = Client(args.base_url, timeout=4)
alice, bob, eve, room = client.bootstrap()
key = uuid4().hex
def same_request(_):
response = Client(args.base_url).message(alice, room, "concurrent replay", key=key)
return response["json"], response["headers"]["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
responses = list(pool.map(same_request, range(30)))
check(
len({row[0]["id"] for row in responses}) == 1, "Cross-replica replay duplicated a message"
)
check(len({row[1] for row in responses}) == 2, "Concurrent replay did not cross real replicas")
result["concurrent_replays"] = 30
result["concurrent_effects"] = 1
try:
for selected in baseline:
selected = instance(selected["id"])
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
env = {**os.environ, "TARGET_INSTANCE_ID": selected["instance"]}
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
timeout=10,
)
action_end = time.monotonic()
check(not inspect(selected["id"])["running"], "Chosen process was not stopped")
time.sleep(3)
report = traffic.report(action_start, action_end)
check(report["first_success_after_action_seconds"] <= 10, "Failover took too long")
check(selected["instance"] not in report["tail_instances"], "Stopped API still serves")
report["stopped"] = selected
traffic.verify_messages(client, bob, room)
result["failures"].append(report)
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
wait_lb(args.base_url, 2)
print("Verified failure and restoration of one API", flush=True)
with Traffic(args.base_url, alice, room) as traffic:
time.sleep(1)
action_start = time.monotonic()
replaced = subprocess.run(
[
sys.executable,
str(ROOT / "scripts/rolling-update.py"),
"--base-url",
args.base_url,
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
timeout=120,
)
action_end = time.monotonic()
time.sleep(3)
traffic.verify_messages(client, bob, room)
result["rolling"] = traffic.report(action_start, action_end)
result["rolling"]["replacements"] = json.loads(replaced.stdout)
result["rolling"]["confirmed_messages_preserved"] = True
print("Verified sequential replacement under traffic", flush=True)
(ROOT / "evidence/failover.txt").write_text(public_test("test-failover", args.base_url))
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
wait_lb(args.base_url, 2)
before = {
service: inspect(compose("ps", "-q", service).strip())
for service in ("postgres", "valkey", "s3", "lb")
}
(ROOT / "evidence/persistence.txt").write_text(
public_test("test-persistence", args.base_url)
)
after = {service: inspect(compose("ps", "-q", service).strip()) for service in before}
check(
all(before[s]["started_at"] != after[s]["started_at"] for s in before),
"Persistence hook did not restart all state/LB services",
)
result["persistence"] = {"before": before, "after": after, "same_token_public_test": True}
result["final_instances"] = instances()
logs = compose("logs", "--no-color", "api", "lb")
check(
not any(
a["token"] in logs or a["registration"]["password"] in logs
for a in (alice, bob, eve)
),
"Credential appeared in logs",
)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print("PASS: real replica failures, replay, rolling replacement and persistence")
finally:
for value in compose("ps", "-aq", "api").split():
if not inspect(value)["running"]:
subprocess.run(["docker", "start", value], check=True, capture_output=True)
if __name__ == "__main__":
main()
+424
View File
@@ -0,0 +1,424 @@
#!/usr/bin/env python3
"""Private S3, 10 concurrent proxy uploads, failure recovery and reconciliation cost."""
import argparse
import hashlib
import io
import json
import os
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
from botocore.exceptions import ClientError
from cluster import compose, inspect, instances, wait_lb
from PIL import Image
from storage_lab import ROOT, connection, store
from messenger.reconcile import Reconciler
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
def put_orphan(admin):
key = "originals/" + str(uuid4())
admin.put(key, io.BytesIO(PNG), len(PNG), "image/png", hashlib.sha256(PNG).hexdigest())
return key
def seed(path):
admin = store("root")
try:
key = put_orphan(admin)
path.write_text(
json.dumps({"orphan": key, "created_at": datetime.now(UTC).isoformat()}) + "\n"
)
print("Seeded one orphan; its real S3 LastModified must age 300 seconds.")
finally:
admin.close()
def memory(cids):
result = {}
for cid in cids:
raw = subprocess.check_output(
[
"docker",
"exec",
cid,
"sh",
"-c",
"cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak "
"/sys/fs/cgroup/cpu.stat",
],
text=True,
)
lines = raw.splitlines()
current, peak = map(int, lines[:2])
counters = {k: int(v) for k, v in (line.split() for line in lines[2:])}
result[cid] = {"current_bytes": current, "peak_bytes": peak, "cpu": counters}
return result
def pressure(client, actor, chat):
results = []
for side in (600, 1700):
image = Image.frombytes("RGB", (side, side), os.urandom(side * side * 3))
output = io.BytesIO()
image.save(output, format="PNG", compress_level=0)
data = output.getvalue()
check(len(data) <= 10485760, "Fixture accidentally oversized")
# Recreate API only, so kernel memory.peak includes this trial and startup.
compose("up", "-d", "--no-deps", "--force-recreate", "--scale", "api=2", "api")
wait_lb(count=2, base=client.base)
cids = [row["id"] for row in instances()]
resource_started = time.monotonic()
baseline = memory(cids)
import threading
barrier = threading.Barrier(10)
def one(i, data=data, barrier=barrier):
c = Client(client.base, timeout=30)
barrier.wait(timeout=10)
started = time.monotonic()
value = c.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=load-{i}.png",
expected=201,
actor=actor,
raw=data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
check(value["json"]["sha256"] == hashlib.sha256(data).hexdigest(), "Checksum mismatch")
return {
"duration_ms": (time.monotonic() - started) * 1000,
"instance": value["headers"]["X-Instance-Id"],
"id": value["json"]["id"],
}
started = time.monotonic()
with ThreadPoolExecutor(max_workers=10) as pool:
values = list(pool.map(one, range(10)))
elapsed = time.monotonic() - started
peak = memory(cids)
resource_seconds = time.monotonic() - resource_started
cpu_cores = (
sum(peak[k]["cpu"]["usage_usec"] - baseline[k]["cpu"]["usage_usec"] for k in cids)
/ 1e6
/ resource_seconds
)
values_ms = sorted(row["duration_ms"] for row in values)
results.append(
{
"clients": 10,
"bytes_each": len(data),
"pixels": side * side,
"duration_seconds": elapsed,
"successful_rps": 10 / elapsed,
"resource_window_seconds": resource_seconds,
"mean_api_cpu_cores_including_probes": cpu_cores,
"errors": 0,
"p50_ms": values_ms[4],
"p95_ms": values_ms[9],
"p99_ms": values_ms[9],
"before": baseline,
"after": peak,
"responses": values,
}
)
print(f"PASS: 10 simultaneous uploads, {len(data)} bytes each", flush=True)
return results
def private(admin, api, chat, actor, client):
value, _ = client.attachment(actor, chat)
with connection() as conn:
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (value["id"],)).fetchone()
# HTTP from a different container, no Authorization, no signature, no API.
code = (
"import urllib.request,urllib.error; "
f'u="http://s3:9000/{api.bucket}/{row["object_key"]}"; '
"\ntry:\n r=urllib.request.urlopen(u); print(r.status)"
"\nexcept urllib.error.HTTPError as e: print(e.code)"
)
cid = instances()[0]["id"]
status = int(subprocess.check_output(["docker", "exec", cid, "python", "-c", code], text=True))
check(status == 403, "Anonymous S3 GET succeeded")
for method, kwargs in [
("delete_object", {"Bucket": api.bucket, "Key": row["object_key"]}),
("list_objects_v2", {"Bucket": "unrelated-bucket"}),
]:
try:
getattr(api.client, method)(**kwargs)
except ClientError as exc:
check(exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Expected IAM 403")
else:
raise RuntimeError("API identity exceeded its policy")
return {
"anonymous_get_status": status,
"api_delete_denied": True,
"other_bucket_denied": True,
"attachment": value["id"],
}
def cross_replica(client, actor, reader, chat):
value = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=switch.png",
expected=201,
actor=actor,
raw=PNG,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
alias = value["headers"]["X-Instance-Id"]
selected = next(row for row in instances() if row["instance"] == alias)
try:
subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
check=True,
capture_output=True,
env={**os.environ, "TARGET_INSTANCE_ID": alias},
timeout=10,
)
stopped = inspect(selected["id"])
check(not stopped["running"], "Uploading process was not stopped")
time.sleep(1)
result = client.call(
"GET", "/api/v1/attachments/" + value["json"]["id"] + "/content", actor=reader
)
check(
result["bytes"] == PNG and result["headers"]["X-Instance-Id"] != alias,
"Another API failed to serve original",
)
return {
"uploader": selected,
"after_stop": stopped,
"downloader_instance": result["headers"]["X-Instance-Id"],
"original_preserved": True,
}
finally:
subprocess.run(["docker", "start", selected["id"]], check=True, capture_output=True)
wait_lb(base=client.base, count=2)
def interruption(client, actor, chat):
with connection() as conn:
before = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"]
# Close in the middle of a correctly declared body; no durable intent must exist yet.
s = socket.create_connection(("127.0.0.1", 8084), timeout=3)
header = (
f"POST /api/v1/chats/{chat['id']}/attachments?filename=interrupted.png HTTP/1.1\r\n"
"Host: localhost\r\nContent-Type: image/png\r\nContent-Length: 1048576\r\n"
f"Authorization: Bearer {actor['token']}\r\nConnection: close\r\n\r\n"
)
s.sendall(header.encode() + PNG[:32])
s.close()
time.sleep(0.5)
with connection() as conn:
after = conn.execute("SELECT count(*) AS n FROM attachments").fetchone()["n"]
check(before == after, "Interrupted inbound body created an intent")
return {"before_rows": before, "after_rows": after, "aborted_body_no_intent": True}
def outage(client, actor, chat):
att, _ = client.attachment(actor, chat)
before = inspect(compose("ps", "-q", "s3").strip())
compose("stop", "s3")
started = time.monotonic()
try:
statuses = []
for method, path, raw in [
("GET", "/api/v1/attachments/" + att["id"] + "/content", None),
("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=unavailable.png", PNG),
]:
request = urllib.request.Request(
client.base + path,
data=raw,
method=method,
headers={"Authorization": "Bearer " + actor["token"], "Content-Type": "image/png"},
)
try:
response = urllib.request.urlopen(request, timeout=8)
except urllib.error.HTTPError as exc:
response = exc
with response:
statuses.append(response.code)
check(response.code == 503, "S3 outage must produce 503")
failed = len(statuses)
finally:
compose("start", "s3")
wait_lb(base=client.base, count=2)
after = inspect(compose("ps", "-q", "s3").strip())
actual = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=actor)[
"bytes"
]
check(actual == PNG, "S3 restart lost original")
return {
"before": before,
"after": after,
"failed_operations": failed,
"statuses": statuses,
"stop_start_and_ready_seconds": time.monotonic() - started,
"original_preserved": True,
}
def reconciliation(seed_path, admin, reconciler, client, actor, chat):
saved = json.loads(seed_path.read_text())
orphan = saved["orphan"]
head = admin.head(orphan)
check(head is not None, "Seed orphan is missing")
age = time.time() - head["LastModified"].timestamp()
check(age >= 300, "Seed must be at least 300s old; keep it while other checks run")
fresh = put_orphan(admin)
used, _ = client.attachment(actor, chat)
client.message(actor, chat, "used attachment", attachments=[used["id"]])
with connection() as conn:
conn.execute(
"UPDATE attachments SET updated_at=clock_timestamp()-interval '10 minutes' WHERE id=%s",
(used["id"],),
)
dry = Reconciler(conn, reconciler).run()
check(
any(
a["action"] == "delete_orphan" and a["id"] == orphan.split("/")[1]
for a in dry["actions"]
),
"Dry run did not detect old orphan",
)
check(admin.head(orphan) is not None, "Dry-run mutated S3")
applied = Reconciler(conn, reconciler, apply=True).run()
repeated = Reconciler(conn, reconciler, apply=True).run()
check(admin.head(orphan) is None and admin.head(fresh) is not None, "Grace safety failed")
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (used["id"],)).fetchone()
check(
row["storage_state"] == "ready" and admin.head(row["object_key"]) is not None,
"Used attachment was removed",
)
check(repeated["actions"] == [], "Repeated apply failed to converge")
# Remove a USED object's bytes, preserve its metadata/link, and repair after re-upload.
admin.client.delete_object(Bucket=admin.bucket, Key=row["object_key"])
missing_dry = Reconciler(conn, reconciler).run()
check(
conn.execute(
"SELECT storage_state FROM attachments WHERE id=%s", (used["id"],)
).fetchone()["storage_state"]
== "ready",
"Dry run mutated metadata",
)
missing_apply = Reconciler(conn, reconciler, apply=True).run()
check(
conn.execute(
"SELECT storage_state FROM attachments WHERE id=%s", (used["id"],)
).fetchone()["storage_state"]
== "missing",
"Missing original was not marked",
)
check(
conn.execute(
"SELECT count(*) AS n FROM message_attachments WHERE attachment_id=%s",
(used["id"],),
).fetchone()["n"]
== 1,
"A used relation was removed",
)
admin.put(
row["object_key"],
io.BytesIO(PNG),
len(PNG),
"image/png",
hashlib.sha256(PNG).hexdigest(),
)
healing = Reconciler(conn, reconciler, apply=True).run()
check(
client.call("GET", "/api/v1/attachments/" + used["id"] + "/content", actor=actor)[
"bytes"
]
== PNG,
"Restored original is wrong",
)
return {
"old_orphan_age_seconds": age,
"dry_run": dry,
"apply": applied,
"repeat": repeated,
"missing_dry_run": missing_dry,
"missing_apply": missing_apply,
"healing": healing,
"fresh_preserved": True,
"used_metadata_and_link_preserved": True,
}
def scan_cost(admin, reconciler):
results = []
created = []
try:
for target in (0, 100, 1100):
while len(created) < target:
created.append(put_orphan(admin))
repeats = []
for _ in range(3):
with connection() as conn:
repeats.append(Reconciler(conn, reconciler).run())
results.append({"extra_fresh_objects": target, "repeats": repeats})
return results
finally:
for key in created:
admin.client.delete_object(Bucket=admin.bucket, Key=key)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--seed", type=Path)
parser.add_argument("--resume-basics", type=Path)
parser.add_argument("--seed-file", type=Path, default=ROOT / "evidence/storage-seed.json")
parser.add_argument("--output", type=Path, default=ROOT / "evidence/storage.json")
args = parser.parse_args()
if args.seed:
seed(args.seed)
return
if args.output.exists():
raise SystemExit("Choose a new evidence filename")
os.environ["NO_PROXY"] = os.environ["no_proxy"] = "localhost,127.0.0.1"
admin, api, rec = store("root"), store("api"), store("reconciler")
try:
client = Client("http://127.0.0.1:8084", timeout=30)
alice, bob, eve, chat = client.bootstrap()
result = {"started_at": datetime.now(UTC).isoformat()}
if args.resume_basics:
result.update(json.loads(args.resume_basics.read_text()))
else:
result["privacy"] = private(admin, api, chat, alice, client)
result["cross_replica"] = cross_replica(client, alice, bob, chat)
result["interruption"] = interruption(client, alice, chat)
result["uploads"] = pressure(client, alice, chat)
result["outage"] = outage(client, alice, chat)
result["reconciliation"] = reconciliation(args.seed_file, admin, rec, client, alice, chat)
result["scan_cost"] = scan_cost(admin, rec)
result["finished_at"] = datetime.now(UTC).isoformat()
args.output.write_text(json.dumps(result, indent=2) + "\n")
print(
"PASS: private S3, upload memory bound, interruption, outage and reconciler", flush=True
)
finally:
for value in (admin, api, rec):
value.close()
if __name__ == "__main__":
main()
+1
View File
@@ -0,0 +1 @@
"""One-process implementation of the lab 1 messenger contract."""
+24
View File
@@ -0,0 +1,24 @@
import logging
import os
import uvicorn
def main():
level = os.getenv("APP_LOG_LEVEL", "INFO").upper()
logging.basicConfig(level=level, format="%(message)s")
uvicorn.run(
"messenger.app:create_app",
factory=True,
host=os.getenv("APP_HOST", "0.0.0.0"),
port=int(os.getenv("APP_PORT", "8080")),
workers=1,
access_log=False,
log_level=level.lower(),
timeout_keep_alive=2,
timeout_graceful_shutdown=5,
)
if __name__ == "__main__":
main()
+337
View File
@@ -0,0 +1,337 @@
"""HTTP adapter. The instructor's OpenAPI, not FastAPI defaults, defines errors."""
import json
import logging
import os
import re
from contextlib import asynccontextmanager
from time import perf_counter
from typing import Annotated
from urllib.parse import quote
from uuid import uuid4
from fastapi import Depends, FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.exceptions import HTTPException
from starlette.types import ASGIApp, Message, Receive, Scope, Send
from messenger.attachments import Attachments
from messenger.auth import AuthService, Identity, basic_credentials, bearer_digest, unauthorized
from messenger.cache import SessionCache
from messenger.domain import DomainError, User, invalid
from messenger.repository import PostgresRepository
from messenger.schemas import (
AttachmentView,
ChatPage,
ChatView,
CreateChat,
CreateMessage,
CreateUser,
MessagePage,
MessageView,
SessionView,
UserView,
canonical_uuid,
)
from messenger.service import Messenger
from messenger.settings import Settings
from messenger.storage import ObjectStore
logger = logging.getLogger("messenger")
class RequestContext:
def __init__(self, app: ASGIApp, instance_id: str):
self.app = app
self.instance_id = instance_id
async def __call__(self, scope: Scope, receive: Receive, send: Send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
request_id = str(uuid4())
scope.setdefault("state", {})["request_id"] = request_id
start, status, response_started = perf_counter(), 500, False
async def with_id(message: Message):
nonlocal status, response_started
if message["type"] == "http.response.start":
response_started = True
status = message["status"]
message["headers"] = [
pair
for pair in message.get("headers", [])
if pair[0].lower() not in (b"x-request-id", b"x-instance-id")
] + [
(b"x-request-id", request_id.encode()),
(b"x-instance-id", self.instance_id.encode()),
]
await send(message)
try:
await self.app(scope, receive, with_id)
except Exception as exc:
# Do not let Uvicorn print exception values that could contain user data.
if response_started:
raise
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
response = error_response(
Request(scope), 500, "internal_error", "Внутренняя ошибка сервера"
)
await response(scope, receive, with_id)
finally:
route = getattr(scope.get("route"), "path", "unmatched")
logger.info(
json.dumps(
{
"event": "request",
"request_id": request_id,
"method": scope["method"],
"route": route,
"status": status,
"duration_ms": round((perf_counter() - start) * 1000, 3),
}
)
)
def error_response(
request: Request, status: int, code: str, message: str, headers: dict | None = None
) -> JSONResponse:
identity = getattr(request.state, "request_id", str(uuid4()))
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": identity}},
headers={"X-Request-Id": identity, **(headers or {})},
)
def create_app(settings: Settings | None = None) -> FastAPI:
settings = settings or Settings.from_env()
repository = PostgresRepository(
settings.database_url,
settings.pool_size,
settings.idempotency_ttl,
settings.read_autocommit,
)
cache = SessionCache(settings.cache_url)
store = ObjectStore(settings)
authentication = AuthService(repository, cache, settings.session_ttl)
@asynccontextmanager
async def lifespan(app: FastAPI):
repository.open()
app.state.ready = True
logger.info(json.dumps({"event": "started", "environment": os.getenv("APP_ENV", "lab")}))
try:
yield
finally:
app.state.ready = False
repository.close()
cache.close()
store.close()
logger.info(json.dumps({"event": "stopped"}))
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
app.state.instance_id = uuid4().hex
app.add_middleware(RequestContext, instance_id=app.state.instance_id)
app.state.messenger = Messenger(repository)
app.state.attachments = Attachments(repository, app.state.messenger, store, settings)
app.state.repository = repository
app.state.authentication = authentication
app.state.cache = cache
app.state.ready = False
@app.exception_handler(DomainError)
async def domain_error(request: Request, exc: DomainError):
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_error(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "Некорректные поля или JSON")
@app.exception_handler(HTTPException)
async def http_error(request: Request, exc: HTTPException):
code = "not_found" if exc.status_code == 404 else "invalid_request"
return error_response(request, exc.status_code, code, "Некорректный HTTP-запрос")
@app.exception_handler(Exception)
async def unexpected_error(request: Request, exc: Exception):
logger.error(json.dumps({"event": "internal_error", "type": type(exc).__name__}))
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
def service(request: Request) -> Messenger:
return request.app.state.messenger
def token_digest(request: Request) -> bytes:
if len(request.headers.getlist("authorization")) != 1:
raise unauthorized()
return bearer_digest(request.headers.get("authorization"))
def identity(
digest: Annotated[bytes, Depends(token_digest)],
) -> Identity:
return authentication.identify(digest)
def actor(current: Annotated[Identity, Depends(identity)]) -> User:
return current.user
async def credentials(request: Request) -> tuple[str, str]:
if len(request.headers.getlist("authorization")) != 1:
raise unauthorized("Basic")
result = basic_credentials(request.headers.get("authorization"))
if await request.body():
raise invalid("Вход не принимает тело запроса")
return result
def resource_id(chat_id: str) -> str:
try:
return canonical_uuid(chat_id)
except ValueError:
raise invalid("Некорректный UUID чата") from None
def pagination(request: Request) -> tuple[int, str | None]:
for key in ("limit", "cursor"):
if len(request.query_params.getlist(key)) > 1:
raise invalid("Параметр пагинации указан несколько раз")
raw = request.query_params.get("limit", "50")
try:
if not re.fullmatch(r"[0-9]+", raw):
raise ValueError
limit = int(raw)
if not 1 <= limit <= 100:
raise ValueError
except ValueError:
raise invalid("limit должен быть целым от 1 до 100") from None
return limit, request.query_params.get("cursor")
def json_input(request: Request):
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if media != "application/json":
raise DomainError(415, "unsupported_media_type", "Требуется application/json")
def message_key(request: Request) -> str:
values = request.headers.getlist("idempotency-key")
if len(values) != 1 or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", values[0]):
raise invalid("Требуется корректный Idempotency-Key")
return values[0]
CurrentUser = Annotated[User, Depends(actor)]
Service = Annotated[Messenger, Depends(service)]
ChatId = Annotated[str, Depends(resource_id)]
Pagination = Annotated[tuple[int, str | None], Depends(pagination)]
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready(request: Request):
if not request.app.state.ready:
raise DomainError(503, "unavailable", "Приложение не готово")
repository.ready()
store.ready()
return {"status": "ok"}
@app.post(
"/api/v1/users",
status_code=201,
response_model=UserView,
dependencies=[Depends(json_input)],
)
def register(body: CreateUser):
return authentication.register(body.username, body.display_name, body.password)
@app.post("/api/v1/auth/sessions", status_code=201, response_model=SessionView)
def login(value: Annotated[tuple[str, str], Depends(credentials)]):
return authentication.login(*value)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(current: Annotated[Identity, Depends(identity)]):
authentication.logout(current)
return Response(status_code=204)
@app.get("/api/v1/users/me", response_model=UserView)
def me(current: CurrentUser):
return current
@app.post(
"/api/v1/chats",
status_code=201,
response_model=ChatView,
dependencies=[Depends(json_input)],
)
def create_chat(body: CreateChat, current: CurrentUser, messenger: Service):
return messenger.create_chat(current, body.title, body.member_ids)
@app.get("/api/v1/chats", response_model=ChatPage)
def chats(current: CurrentUser, messenger: Service, page: Pagination):
return messenger.chats(current, *page)
@app.get("/api/v1/chats/{chat_id}", response_model=ChatView)
def chat(current: CurrentUser, chat: ChatId, messenger: Service):
return messenger.chat(current, chat)
@app.post(
"/api/v1/chats/{chat_id}/messages",
status_code=201,
response_model=MessageView,
dependencies=[Depends(json_input)],
)
def send_message(
body: CreateMessage,
current: CurrentUser,
chat: ChatId,
messenger: Service,
key: Annotated[str, Depends(message_key)],
):
return messenger.send(current, chat, body.text, key, tuple(body.attachment_ids))
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
def messages(current: CurrentUser, chat: ChatId, messenger: Service, page: Pagination):
return messenger.messages(current, chat, *page)
def attachment_id(attachment_id: str) -> str:
try:
return canonical_uuid(attachment_id)
except ValueError:
raise invalid("Некорректный UUID вложения") from None
AttachmentId = Annotated[str, Depends(attachment_id)]
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201, response_model=AttachmentView)
async def upload(request: Request, current: CurrentUser, chat: ChatId):
return await app.state.attachments.upload(request, current, chat)
@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentView)
def attachment(current: CurrentUser, id: AttachmentId):
return app.state.attachments.metadata(current, id)
@app.get("/api/v1/attachments/{attachment_id}/content")
def content(request: Request, current: CurrentUser, id: AttachmentId):
row = app.state.attachments.record(current, id)
values = request.query_params.getlist("variant")
if len(values) > 1 or values and values[0] != "original":
raise invalid("Некорректный вариант вложения")
body = store.download(row)
def chunks():
try:
while data := body.read(65536):
yield data
finally:
body.close()
return StreamingResponse(
chunks(),
media_type=row["content_type"],
headers={
"Content-Length": str(row["size_bytes"]),
"Content-Disposition": "attachment; filename*=UTF-8''"
+ quote(row["filename"], safe=""),
"Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff",
},
)
return app
+124
View File
@@ -0,0 +1,124 @@
"""Bounded binary upload. Durable intent precedes S3; ready follows both stores."""
import asyncio
import hashlib
import tempfile
import unicodedata
import warnings
from anyio import to_thread
from PIL import Image, UnidentifiedImageError
from starlette.requests import ClientDisconnect
from messenger.domain import DomainError, invalid, new_id, not_found
from messenger.repository import attachment_value
Image.MAX_IMAGE_PIXELS = 40_000_000
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
def filename(value):
if (
not value
or not value.strip()
or len(value) > 255
or any(c in "/\\" or unicodedata.category(c).startswith("C") for c in value)
):
raise invalid("Некорректное имя файла")
return value
def image_type(file, maximum):
try:
file.seek(0)
with Image.open(file) as image:
if image.width * image.height > maximum:
raise DomainError(413, "payload_too_large", "Слишком много пикселей")
if image.format not in ("PNG", "JPEG"):
raise ValueError
media = "image/png" if image.format == "PNG" else "image/jpeg"
image.verify()
file.seek(0)
with Image.open(file) as image:
image.load() # Reject truncated pixel streams, not just an intact header.
file.seek(0)
return media
except Image.DecompressionBombError, Image.DecompressionBombWarning:
raise DomainError(413, "payload_too_large", "Слишком много пикселей") from None
except UnidentifiedImageError, OSError, ValueError, SyntaxError:
raise DomainError(415, "unsupported_media_type", "Повреждённое изображение") from None
class Attachments:
def __init__(self, repository, messenger, store, settings):
self.repository, self.messenger, self.store = repository, messenger, store
self.settings = settings
self.slots = asyncio.Semaphore(settings.upload_slots)
async def upload(self, request, actor, chat_id):
await to_thread.run_sync(self.messenger.chat, actor, chat_id)
values = request.query_params.getlist("filename")
if len(values) != 1:
raise invalid("Требуется одно имя файла")
name = filename(values[0])
media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if media not in ("image/png", "image/jpeg"):
raise DomainError(415, "unsupported_media_type", "Требуется PNG или JPEG")
length = request.headers.get("content-length")
if length is not None:
if not length.isascii() or not length.isdecimal():
raise invalid("Некорректный Content-Length")
if int(length) > self.settings.max_upload_bytes:
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
try:
await asyncio.wait_for(self.slots.acquire(), timeout=5)
except TimeoutError:
raise DomainError(429, "rate_limited", "Слишком много загрузок") from None
try:
with tempfile.TemporaryFile() as file:
size, digest = 0, hashlib.sha256()
try:
async with asyncio.timeout(15):
async for chunk in request.stream():
size += len(chunk)
if size > self.settings.max_upload_bytes:
raise DomainError(413, "payload_too_large", "Файл больше 10 MiB")
digest.update(chunk)
await to_thread.run_sync(file.write, chunk)
except TimeoutError:
raise DomainError(400, "invalid_request", "Время загрузки истекло") from None
except ClientDisconnect:
raise invalid("Загрузка прервана") from None
actual = await to_thread.run_sync(image_type, file, self.settings.max_image_pixels)
if actual != media:
raise DomainError(415, "unsupported_media_type", "MIME не совпадает с файлом")
id = new_id()
row = await to_thread.run_sync(
self.repository.begin_attachment,
id,
chat_id,
actor.id,
name,
actual,
size,
digest.hexdigest(),
)
# Leave the committed intent on every ambiguous failure. Reconciler owns recovery.
await to_thread.run_sync(
self.store.put, row["object_key"], file, size, actual, digest.hexdigest()
)
return await to_thread.run_sync(self.repository.finish_attachment, id)
finally:
self.slots.release()
def record(self, actor, id):
row = self.repository.attachment(id)
if row is None:
raise not_found()
self.messenger.chat(actor, str(row["chat_id"]))
if row["storage_state"] != "ready":
raise DomainError(409, "conflict", "Вложение пока недоступно")
return row
def metadata(self, actor, id):
return attachment_value(self.record(actor, id))
+127
View File
@@ -0,0 +1,127 @@
"""Argon2id passwords and opaque sessions. Cache hits never authorize a request."""
import base64
import binascii
import hashlib
import re
import secrets
from contextlib import contextmanager
from dataclasses import dataclass
from threading import BoundedSemaphore
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError
from argon2.profiles import RFC_9106_LOW_MEMORY
from messenger.cache import SessionCache
from messenger.domain import DomainError, User
from messenger.repository import PostgresRepository
def unauthorized(scheme: str = "Bearer") -> DomainError:
challenge = scheme + ' realm="messenger"'
if scheme == "Basic":
challenge += ', charset="UTF-8"'
return DomainError(
401,
"unauthorized",
"Требуется действительная идентификация",
{"WWW-Authenticate": challenge},
)
def basic_credentials(header: str | None) -> tuple[str, str]:
try:
if header is None or len(header) > 2048:
raise ValueError
scheme, encoded = header.split(" ", 1)
if scheme.lower() != "basic":
raise ValueError
raw = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = raw.split(":", 1)
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
raise ValueError
if not 12 <= len(password) <= 128 or not password.strip():
raise ValueError
return username, password
except ValueError, UnicodeError, binascii.Error:
raise unauthorized("Basic") from None
def bearer_digest(header: str | None) -> bytes:
if header:
parts = header.split(" ")
if len(parts) == 2 and parts[0].lower() == "bearer":
if re.fullmatch(r"[A-Za-z0-9_-]{43}", parts[1]):
return hashlib.sha256(parts[1].encode()).digest()
raise unauthorized()
@dataclass(frozen=True)
class Identity:
user: User
token_hash: bytes
@dataclass(frozen=True)
class Session:
token: str
token_type: str
expires_at: str
user: User
class AuthService:
def __init__(self, repository: PostgresRepository, cache: SessionCache, ttl: int):
self.repository, self.cache, self.ttl = repository, cache, ttl
self.hasher = PasswordHasher.from_parameters(RFC_9106_LOW_MEMORY)
self.kdf_slots = BoundedSemaphore(2)
self.dummy_hash = self.hasher.hash(secrets.token_urlsafe(32))
@contextmanager
def kdf(self):
if not self.kdf_slots.acquire(timeout=0.5):
raise DomainError(503, "unavailable", "Обработка пароля временно занята")
try:
yield
finally:
self.kdf_slots.release()
def register(self, username: str, display_name: str, password: str) -> User:
with self.kdf():
encoded = self.hasher.hash(password)
return self.repository.add_user(username, display_name, encoded)
def login(self, username: str, password: str) -> Session:
credentials = self.repository.credentials(username)
encoded = credentials[1] if credentials else self.dummy_hash
try:
with self.kdf():
self.hasher.verify(encoded, password)
except VerificationError, InvalidHashError:
raise unauthorized("Basic") from None
if credentials is None:
raise unauthorized("Basic")
token = secrets.token_urlsafe(32)
digest = hashlib.sha256(token.encode()).digest()
user = credentials[0]
expires = self.repository.create_session(digest, user.id, self.ttl)
return Session(token, "Bearer", expires, user)
def identify(self, digest: bytes) -> Identity:
session = self.repository.active_session(digest)
if session is None:
raise unauthorized()
user_id = str(session["user_id"])
user = self.cache.get(digest, user_id)
if user is None:
user = self.repository.get_user(user_id)
if user is None:
raise unauthorized()
ttl = int((session["expires_at"] - session["checked_at"]).total_seconds())
self.cache.put(digest, user, ttl)
return Identity(user, digest)
def logout(self, identity: Identity):
self.repository.revoke_session(identity.token_hash)
self.cache.delete(identity.token_hash)
+57
View File
@@ -0,0 +1,57 @@
"""Disposable session-user cache; PostgreSQL always decides token validity."""
import json
import logging
from dataclasses import asdict
from redis import Redis
from redis.backoff import NoBackoff
from redis.exceptions import RedisError
from redis.retry import Retry
from messenger.domain import User
logger = logging.getLogger("messenger")
class SessionCache:
def __init__(self, url: str):
self.client = Redis.from_url(
url,
socket_connect_timeout=0.2,
socket_timeout=0.2,
max_connections=8,
decode_responses=True,
retry_on_timeout=False,
retry=Retry(NoBackoff(), 0),
)
@staticmethod
def key(digest: bytes) -> str:
return "session-user:" + digest.hex()
def get(self, digest: bytes, user_id: str) -> User | None:
try:
raw = self.client.get(self.key(digest))
if raw:
value = User(**json.loads(raw))
if value.id == user_id:
return value
except RedisError, ValueError, TypeError:
logger.warning('{"event":"session_cache_unavailable","operation":"get"}')
return None
def put(self, digest: bytes, user: User, ttl: int):
try:
self.client.set(self.key(digest), json.dumps(asdict(user)), ex=max(1, ttl))
except RedisError:
logger.warning('{"event":"session_cache_unavailable","operation":"set"}')
def delete(self, digest: bytes):
try:
self.client.delete(self.key(digest))
except RedisError:
logger.warning('{"event":"session_cache_unavailable","operation":"delete"}')
def close(self):
self.client.close()
+41
View File
@@ -0,0 +1,41 @@
"""Versioned keyset position, bound to a list and its resource, never an offset."""
import base64
import binascii
import json
from datetime import UTC, datetime
from uuid import UUID
from messenger.domain import HistoryKey, invalid
def encode_cursor(scope: str, position: HistoryKey) -> str:
value = {"v": 1, "scope": scope, "created_at": position[0], "id": position[1]}
return (
base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode())
.decode()
.rstrip("=")
)
def decode_cursor(value: str | None, scope: str) -> HistoryKey | None:
if value is None:
return None
try:
if not value or len(value) > 1024:
raise ValueError
raw = base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True)
data = json.loads(raw)
if not isinstance(data, dict) or set(data) != {"v", "scope", "created_at", "id"}:
raise ValueError
if type(data["v"]) is not int or data["v"] != 1 or data["scope"] != scope:
raise ValueError
if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"):
raise ValueError
stamp = datetime.fromisoformat(data["created_at"])
if stamp.tzinfo != UTC or not isinstance(data["id"], str):
raise ValueError
identity = str(UUID(data["id"]))
return stamp.isoformat(timespec="microseconds").replace("+00:00", "Z"), identity
except ValueError, TypeError, KeyError, UnicodeError, binascii.Error:
raise invalid("Некорректный cursor или другая область пагинации") from None
+79
View File
@@ -0,0 +1,79 @@
"""Immutable domain values; no dependency on HTTP or a storage technology."""
from dataclasses import dataclass
from datetime import UTC, datetime
from uuid import uuid4
def utc_now() -> str:
return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z")
def new_id() -> str:
return str(uuid4())
class DomainError(Exception):
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid(message: str = "Некорректный запрос") -> DomainError:
return DomainError(400, "invalid_request", message)
def not_found() -> DomainError:
return DomainError(404, "not_found", "Объект не найден")
@dataclass(frozen=True, slots=True)
class User:
id: str
username: str
display_name: str
created_at: str
@dataclass(frozen=True, slots=True)
class Chat:
id: str
title: str
member_ids: tuple[str, ...]
created_at: str
@dataclass(frozen=True, slots=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: str
attachment_ids: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class Attachment:
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
created_at: str
updated_at: str
status: str = "ready"
variants: tuple = ()
error_code: str | None = None
type HistoryKey = tuple[str, str]
def history_key(item: Chat | Message) -> HistoryKey:
return item.created_at, item.id
+46
View File
@@ -0,0 +1,46 @@
"""Transactional, checksummed migrations, serialized across concurrent starts."""
import hashlib
import os
from pathlib import Path
import psycopg
from messenger.settings import secret
def migrate(conninfo: str, directory: Path):
with psycopg.connect(conninfo, connect_timeout=5) as conn:
conn.execute("SELECT pg_advisory_xact_lock(2301002)")
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
"version text PRIMARY KEY, checksum text NOT NULL, "
"applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
)
for path in sorted(directory.glob("*.sql")):
content = path.read_text()
checksum = hashlib.sha256(content.encode()).hexdigest()
current = conn.execute(
"SELECT checksum FROM schema_migrations WHERE version=%s", (path.stem,)
).fetchone()
if current:
if current[0] != checksum:
raise RuntimeError("Applied migration checksum changed: " + path.stem)
continue
conn.execute(content)
conn.execute(
"INSERT INTO schema_migrations (version,checksum) VALUES (%s,%s)",
(path.stem, checksum),
)
print("Applied migration " + path.stem)
def main():
migrate(
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
Path(os.getenv("MIGRATIONS_DIR", "/app/migrations")),
)
if __name__ == "__main__":
main()
+207
View File
@@ -0,0 +1,207 @@
"""Conservative, repeatable PG/S3 reconciliation. Dry-run is the default."""
import argparse
import json
import re
from collections import Counter
from datetime import timedelta
from time import perf_counter
import psycopg
from psycopg.rows import dict_row
from messenger.settings import Settings
from messenger.storage import ObjectStore
OWNED = re.compile(r"originals/[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\Z")
class Reconciler:
def __init__(self, conn, store, *, apply=False, grace=300):
if grace < 300:
raise ValueError("Minimum grace period is 300 seconds")
self.conn, self.store, self.apply, self.grace = conn, store, apply, grace
self.counts, self.actions = Counter(), []
def action(self, kind, id):
self.counts[kind] += 1
if len(self.actions) < 100:
self.actions.append({"action": kind, "id": str(id)})
def head(self, key):
self.counts["s3_head"] += 1
return self.store.head(key)
def objects(self, cutoff):
paginator = self.store.client.get_paginator("list_objects_v2")
for page in paginator.paginate(
Bucket=self.store.bucket, Prefix="originals/", PaginationConfig={"PageSize": 1000}
):
self.counts["s3_list_pages"] += 1
for item in page.get("Contents", []):
self.counts["s3_objects"] += 1
key = item["Key"]
if not OWNED.fullmatch(key):
self.counts["foreign_keys_ignored"] += 1
continue
if item["LastModified"] >= cutoff:
self.counts["fresh_objects_skipped"] += 1
continue
with self.conn.transaction():
existing = self.conn.execute(
"SELECT id FROM attachments WHERE object_key=%s FOR UPDATE",
(key,),
).fetchone()
if existing:
continue
# Re-check immutable generation/age immediately before deleting an orphan.
current = self.head(key)
if (
current is None
or current["LastModified"] >= cutoff
or current["ETag"] != item["ETag"]
or current["ContentLength"] != item["Size"]
):
continue
self.action("delete_orphan", key.removeprefix("originals/"))
if self.apply:
self.store.client.delete_object(Bucket=self.store.bucket, Key=key)
self.counts["s3_delete"] += 1
@staticmethod
def matches(row, head):
return (
head is not None
and head["ContentLength"] == row["size_bytes"]
and head.get("ContentType") == row["content_type"]
and head.get("Metadata", {}).get("sha256") == row["sha256"]
)
def metadata(self, cutoff):
last = "00000000-0000-0000-0000-000000000000"
while True:
batch = self.conn.execute(
"SELECT id FROM attachments WHERE id>%s::uuid ORDER BY id LIMIT 100",
(last,),
).fetchall()
self.counts["pg_metadata_pages"] += 1
if not batch:
break
for value in batch:
id = value["id"]
last = str(id)
self.counts["pg_attachments"] += 1
with self.conn.transaction():
row = self.conn.execute(
"SELECT * FROM attachments WHERE id=%s FOR UPDATE", (id,)
).fetchone()
if row is None:
continue
state = row["storage_state"]
if row["updated_at"] >= cutoff and state != "missing":
self.counts["fresh_metadata_skipped"] += 1
continue
head = self.head(row["object_key"])
valid = self.matches(row, head)
if state == "pending" and head is None:
used = self.conn.execute(
"SELECT 1 FROM message_attachments WHERE attachment_id=%s LIMIT 1",
(id,),
).fetchone()
if used:
self.action("review_used_pending", id)
continue
self.action("delete_abandoned_intent", id)
if self.apply:
self.conn.execute("DELETE FROM attachments WHERE id=%s", (id,))
elif state in ("pending", "missing") and valid:
self.action("restore_ready", id)
if self.apply:
self.conn.execute(
"UPDATE attachments SET storage_state='ready',"
"updated_at=clock_timestamp() WHERE id=%s",
(id,),
)
elif state == "ready" and not valid:
self.action("mark_missing", id)
if self.apply:
self.conn.execute(
"UPDATE attachments SET storage_state='missing',"
"updated_at=clock_timestamp() WHERE id=%s",
(id,),
)
elif state == "pending" and not valid:
self.action("review_invalid_pending", id)
elif state == "missing":
self.counts["still_missing"] += 1
else:
self.counts["ready_preserved"] += 1
def run(self):
started = perf_counter()
if not self.conn.execute("SELECT pg_try_advisory_lock(2301004) AS locked").fetchone()[
"locked"
]:
raise RuntimeError("Another reconciler is active")
try:
cutoff = self.conn.execute("SELECT clock_timestamp() AS now").fetchone()["now"]
cutoff -= timedelta(seconds=self.grace)
self.store.ready() # Permission/network errors are never interpreted as missing bytes.
self.objects(cutoff)
self.metadata(cutoff)
return {
"mode": "apply" if self.apply else "dry-run",
"grace_seconds": self.grace,
"elapsed_seconds": perf_counter() - started,
"counts": dict(self.counts),
"actions": self.actions,
"actions_preview_limit": 100,
}
finally:
self.conn.execute("SELECT pg_advisory_unlock(2301004)")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--apply", action="store_true")
parser.add_argument("--grace-seconds", type=int, default=300)
args = parser.parse_args()
if args.grace_seconds < 300:
parser.error("Grace period must be at least 300 seconds")
store = None
try:
settings = Settings.from_env()
store = ObjectStore(settings)
with psycopg.connect(
settings.database_url,
autocommit=True,
row_factory=dict_row,
connect_timeout=2,
options="-c lock_timeout=1000 "
"-c statement_timeout=2500 -c idle_in_transaction_session_timeout=5000 "
"-c timezone=UTC",
) as conn:
print(
json.dumps(
Reconciler(conn, store, apply=args.apply, grace=args.grace_seconds).run(),
ensure_ascii=False,
)
)
except Exception as exc:
print(
json.dumps(
{
"error": "reconciliation_incomplete",
"type": type(exc).__name__,
"changes_may_have_committed": args.apply,
}
)
)
raise SystemExit(1) from None
finally:
if store:
store.close()
if __name__ == "__main__":
main()
+353
View File
@@ -0,0 +1,353 @@
"""PostgreSQL adapter: bounded connections, parameterized SQL, commit before return."""
import hashlib
import json
from contextlib import contextmanager, nullcontext
from datetime import datetime
from typing import Protocol
import psycopg
from psycopg import errors
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
from messenger.domain import (
Attachment,
Chat,
DomainError,
HistoryKey,
Message,
User,
invalid,
new_id,
)
def stamp(value: datetime) -> str:
return value.isoformat(timespec="microseconds").replace("+00:00", "Z")
def user_value(row) -> User:
return User(str(row["id"]), row["username"], row["display_name"], stamp(row["created_at"]))
def chat_value(row) -> Chat:
return Chat(
str(row["id"]),
row["title"],
tuple(str(x) for x in row["member_ids"]),
stamp(row["created_at"]),
)
def message_value(row) -> Message:
return Message(
str(row["id"]),
str(row["chat_id"]),
str(row["sender_id"]),
row["text"],
stamp(row["created_at"]),
tuple(str(x) for x in row.get("attachment_ids", ())),
)
def attachment_value(row) -> Attachment:
return Attachment(
*(str(row[k]) for k in ("id", "chat_id", "owner_id")),
row["filename"],
row["content_type"],
row["size_bytes"],
row["sha256"],
stamp(row["created_at"]),
stamp(row["updated_at"]),
)
class Repository(Protocol):
def get_user(self, user_id: str) -> User | None: ...
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat: ...
def get_chat(self, chat_id: str) -> Chat | None: ...
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]: ...
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message: ...
def idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
key: str,
attachment_ids: tuple[str, ...] = (),
) -> Message: ...
def page_messages(
self, chat_id: str, after: HistoryKey | None, limit: int
) -> list[Message]: ...
CHAT_COLUMNS = """c.id, c.title, c.created_at,
ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY position) AS member_ids"""
MESSAGE_COLUMNS = """m.*, ARRAY(SELECT attachment_id FROM message_attachments
WHERE message_id=m.id ORDER BY position) AS attachment_ids"""
class PostgresRepository:
def __init__(
self,
conninfo: str,
pool_size: int = 8,
idempotency_ttl: int = 86400,
read_autocommit: bool = True,
):
self.read_autocommit = read_autocommit
self.idempotency_ttl = idempotency_ttl
self.pool = ConnectionPool(
conninfo,
min_size=1,
max_size=pool_size,
timeout=1,
max_waiting=32,
open=False,
check=ConnectionPool.check_connection,
kwargs={
"row_factory": dict_row,
"autocommit": True,
"connect_timeout": 2,
"options": "-c statement_timeout=2500 -c lock_timeout=1000 "
"-c idle_in_transaction_session_timeout=5000 -c timezone=UTC",
},
)
def open(self):
self.pool.open(wait=True, timeout=15)
def close(self):
self.pool.close(timeout=5)
@contextmanager
def connection(self, *, write: bool = False):
try:
with self.pool.connection() as conn:
transaction = (
conn.transaction() if write or not self.read_autocommit else nullcontext()
)
with transaction:
yield conn
except (
psycopg.OperationalError,
psycopg.InterfaceError,
PoolTimeout,
TooManyRequests,
errors.QueryCanceled,
errors.LockNotAvailable,
):
raise DomainError(503, "unavailable", "Хранилище временно недоступно") from None
def ready(self):
with self.connection() as conn:
row = conn.execute(
"SELECT count(*) AS n FROM schema_migrations "
"WHERE version IN ('001_initial','002_idempotency','003_attachments')"
).fetchone()
if row["n"] != 3:
raise DomainError(503, "unavailable", "Схема ещё не готова")
def add_user(self, username: str, display_name: str, password_hash: str) -> User:
try:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO users (id,username,display_name,password_hash) "
"VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), username, display_name, password_hash),
).fetchone()
return user_value(row)
except errors.UniqueViolation:
raise DomainError(409, "conflict", "Имя пользователя уже занято") from None
def credentials(self, username: str):
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
return (user_value(row), row["password_hash"]) if row else None
def get_user(self, user_id: str) -> User | None:
with self.connection() as conn:
row = conn.execute("SELECT * FROM users WHERE id=%s", (user_id,)).fetchone()
return user_value(row) if row else None
def add_chat(self, title: str, member_ids: tuple[str, ...]) -> Chat:
try:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING *", (new_id(), title)
).fetchone()
with conn.cursor() as cur:
cur.executemany(
"INSERT INTO chat_members (chat_id,user_id,position) VALUES (%s,%s,%s)",
[(row["id"], member, i) for i, member in enumerate(member_ids)],
)
return Chat(str(row["id"]), title, member_ids, stamp(row["created_at"]))
except errors.ForeignKeyViolation:
raise invalid("Указан неизвестный участник") from None
def get_chat(self, chat_id: str) -> Chat | None:
with self.connection() as conn:
row = conn.execute(
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id=%s", (chat_id,)
).fetchone()
return chat_value(row) if row else None
def page_chats(self, user_id: str, after: HistoryKey | None, limit: int) -> list[Chat]:
condition = " AND (c.created_at,c.id) > (%s::timestamptz,%s::uuid)" if after else ""
params = (user_id, *after, limit) if after else (user_id, limit)
with self.connection() as conn:
rows = conn.execute(
"SELECT " + CHAT_COLUMNS + " FROM chats c WHERE EXISTS "
"(SELECT 1 FROM chat_members m WHERE m.chat_id=c.id AND m.user_id=%s)"
+ condition
+ " ORDER BY c.created_at,c.id LIMIT %s",
params,
).fetchall()
return [chat_value(row) for row in rows]
def add_message(self, chat_id: str, sender_id: str, text: str) -> Message:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), chat_id, sender_id, text),
).fetchone()
return message_value(row)
def page_messages(self, chat_id: str, after: HistoryKey | None, limit: int) -> list[Message]:
condition = " AND (m.created_at,m.id) > (%s::timestamptz,%s::uuid)" if after else ""
params = (chat_id, *after, limit) if after else (chat_id, limit)
with self.connection() as conn:
rows = conn.execute(
"SELECT "
+ MESSAGE_COLUMNS
+ " FROM messages m WHERE m.chat_id=%s"
+ condition
+ " ORDER BY m.created_at,m.id LIMIT %s",
params,
).fetchall()
return [message_value(row) for row in rows]
def idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
key: str,
attachment_ids: tuple[str, ...] = (),
) -> Message:
scope = json.dumps([sender_id, "POST", chat_id, key], separators=(",", ":"))
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True)
payload = json.dumps(
{"text": text, "attachment_ids": list(attachment_ids)},
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
)
digest = hashlib.sha256(payload.encode()).digest()
with self.connection(write=True) as conn:
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
previous = conn.execute(
"SELECT r.payload_hash," + MESSAGE_COLUMNS + " FROM message_requests r "
"JOIN messages m ON m.id=r.response_message_id "
"WHERE r.user_id=%s AND r.http_method='POST' AND r.chat_id=%s "
"AND r.idempotency_key=%s AND r.expires_at > clock_timestamp()",
(sender_id, chat_id, key),
).fetchone()
if previous:
if previous["payload_hash"] != digest:
raise DomainError(409, "conflict", "Ключ уже использован с другим сообщением")
return message_value(previous)
conn.execute(
"DELETE FROM message_requests WHERE user_id=%s AND http_method='POST' "
"AND chat_id=%s AND idempotency_key=%s",
(sender_id, chat_id, key),
)
if attachment_ids:
rows = conn.execute(
"SELECT id,storage_state FROM attachments WHERE chat_id=%s "
"AND id=ANY(%s::uuid[]) ORDER BY id FOR UPDATE",
(chat_id, list(attachment_ids)),
).fetchall()
if len(rows) != len(attachment_ids):
raise DomainError(404, "not_found", "Объект не найден")
if any(row["storage_state"] != "ready" for row in rows):
raise DomainError(409, "conflict", "Вложение пока недоступно")
message = message_value(
conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text) "
"VALUES (%s,%s,%s,%s) RETURNING *",
(new_id(), chat_id, sender_id, text),
).fetchone()
)
for position, attachment in enumerate(attachment_ids):
conn.execute(
"INSERT INTO message_attachments VALUES (%s,%s,%s,%s)",
(message.id, chat_id, attachment, position),
)
message = Message(
message.id,
message.chat_id,
message.sender_id,
message.text,
message.created_at,
attachment_ids,
)
self._save_request(conn, sender_id, chat_id, key, digest, message.id)
return message
def _save_request(self, conn, user_id, chat_id, key, digest, message_id):
conn.execute(
"INSERT INTO message_requests "
"(user_id,chat_id,idempotency_key,payload_hash,response_message_id,expires_at) "
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+make_interval(secs => %s))",
(user_id, chat_id, key, digest, message_id, self.idempotency_ttl),
)
def create_session(self, token_hash: bytes, user_id: str, ttl: int) -> str:
with self.connection(write=True) as conn:
row = conn.execute(
"INSERT INTO sessions (token_hash,user_id,expires_at) "
"VALUES (%s,%s,clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
(token_hash, user_id, ttl),
).fetchone()
return stamp(row["expires_at"])
def active_session(self, token_hash: bytes):
with self.connection() as conn:
return conn.execute(
"SELECT user_id,expires_at,clock_timestamp() AS checked_at FROM sessions "
"WHERE token_hash=%s AND expires_at > clock_timestamp()",
(token_hash,),
).fetchone()
def revoke_session(self, token_hash: bytes):
with self.connection(write=True) as conn:
conn.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash,))
def begin_attachment(self, id, chat_id, owner_id, filename, content_type, size, digest):
with self.connection(write=True) as conn:
return conn.execute(
"INSERT INTO attachments "
"(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,"
"object_key,storage_state) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending') RETURNING *",
(id, chat_id, owner_id, filename, content_type, size, digest, "originals/" + id),
).fetchone()
def finish_attachment(self, id):
with self.connection(write=True) as conn:
row = conn.execute(
"UPDATE attachments SET storage_state='ready',updated_at=clock_timestamp() "
"WHERE id=%s AND storage_state='pending' RETURNING *",
(id,),
).fetchone()
if not row:
raise DomainError(503, "unavailable", "Загрузка не завершена")
return attachment_value(row)
def attachment(self, id):
with self.connection() as conn:
return conn.execute("SELECT * FROM attachments WHERE id=%s", (id,)).fetchone()
+120
View File
@@ -0,0 +1,120 @@
"""Strict lab 4 wire types; password never belongs to a response User."""
import re
from typing import Annotated
from uuid import UUID
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator
def canonical_uuid(value: str) -> str:
if not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value):
raise ValueError("Ожидается UUID")
return str(UUID(value))
def nonblank(value: str) -> str:
if not value.strip() or any(0xD800 <= ord(char) <= 0xDFFF for char in value):
raise ValueError("Ожидается непустая строка Unicode")
return value
type Identifier = Annotated[str, AfterValidator(canonical_uuid)]
type ShortText = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(nonblank)]
class Input(BaseModel):
model_config = ConfigDict(strict=True, extra="forbid")
class CreateUser(Input):
username: str = Field(min_length=3, max_length=32)
display_name: ShortText
password: Annotated[str, Field(min_length=12, max_length=128), AfterValidator(nonblank)]
@field_validator("username")
@classmethod
def valid_username(cls, value: str) -> str:
if not re.fullmatch(r"[a-z0-9_]{3,32}", value):
raise ValueError("Недопустимое имя пользователя")
return value
class CreateChat(Input):
title: ShortText
member_ids: list[Identifier] = Field(min_length=1, max_length=100)
class CreateMessage(Input):
text: str = Field(max_length=4000)
attachment_ids: list[Identifier] = Field(default_factory=list, max_length=10)
@model_validator(mode="after")
def valid_content(self):
if (
not self.text.strip()
and not self.attachment_ids
or any(0xD800 <= ord(c) <= 0xDFFF for c in self.text)
or len(set(self.attachment_ids)) != len(self.attachment_ids)
):
raise ValueError("Некорректное содержимое сообщения")
return self
class View(BaseModel):
model_config = ConfigDict(from_attributes=True, extra="forbid")
class UserView(View):
id: str
username: str
display_name: str
created_at: str
class SessionView(View):
token: str
token_type: str
expires_at: str
user: UserView
class ChatView(View):
id: str
title: str
member_ids: list[str]
created_at: str
class MessageView(View):
attachment_ids: list[str]
id: str
chat_id: str
sender_id: str
text: str
created_at: str
class ChatPage(View):
items: list[ChatView]
next_cursor: str | None
class MessagePage(View):
items: list[MessageView]
next_cursor: str | None
class AttachmentView(View):
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
status: str
variants: list
error_code: str | None
created_at: str
updated_at: str
+52
View File
@@ -0,0 +1,52 @@
"""Use cases shared by HTTP and tests. Authorization precedes resource access."""
from dataclasses import dataclass
from messenger.cursor import decode_cursor, encode_cursor
from messenger.domain import Chat, Message, User, history_key, invalid, not_found
from messenger.repository import Repository
@dataclass(frozen=True)
class Page[T]:
items: list[T]
next_cursor: str | None
class Messenger:
def __init__(self, repository: Repository):
self.repository = repository
def create_chat(self, actor: User, title: str, member_ids: list[str]) -> Chat:
if len(member_ids) != len(set(member_ids)) or actor.id not in member_ids:
raise invalid("Участники не должны повторяться; вызывающий должен входить в чат")
return self.repository.add_chat(title, tuple(member_ids))
def chat(self, actor: User, chat_id: str) -> Chat:
chat = self.repository.get_chat(chat_id)
if chat is None or actor.id not in chat.member_ids:
raise not_found()
return chat
def chats(self, actor: User, limit: int, cursor: str | None) -> Page[Chat]:
scope = "chats:" + actor.id
after = decode_cursor(cursor, scope)
return self._page(self.repository.page_chats(actor.id, after, limit + 1), limit, scope)
def send(
self, actor: User, chat_id: str, text: str, key: str, attachments: tuple[str, ...] = ()
) -> Message:
self.chat(actor, chat_id)
return self.repository.idempotent_message(chat_id, actor.id, text, key, attachments)
def messages(self, actor: User, chat_id: str, limit: int, cursor: str | None) -> Page[Message]:
self.chat(actor, chat_id)
scope = "messages:" + chat_id
after = decode_cursor(cursor, scope)
return self._page(self.repository.page_messages(chat_id, after, limit + 1), limit, scope)
@staticmethod
def _page[T: Chat | Message](items: list[T], limit: int, scope: str) -> Page[T]:
visible = items[:limit]
cursor = encode_cursor(scope, history_key(visible[-1])) if len(items) > limit else None
return Page(visible, cursor)
+57
View File
@@ -0,0 +1,57 @@
"""Explicit configuration; credentials are read from untracked secret files."""
import os
from dataclasses import dataclass, field
from pathlib import Path
def secret(name: str, default: str) -> str:
return Path(os.getenv(name, default)).read_text().strip()
@dataclass(frozen=True)
class Settings:
database_url: str = field(repr=False)
cache_url: str = field(repr=False)
session_ttl: int = 3600
pool_size: int = 8
idempotency_ttl: int = 86400
read_autocommit: bool = True
s3_endpoint: str = "http://127.0.0.1:59004"
s3_region: str = "us-east-1"
s3_bucket: str = "messenger-attachments"
s3_access_key: str = field(default="", repr=False)
s3_secret_key: str = field(default="", repr=False)
max_upload_bytes: int = 10485760
max_image_pixels: int = 40000000
upload_slots: int = 1
@classmethod
def from_env(cls):
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
size = int(os.getenv("DATABASE_POOL_SIZE", "8"))
idempotency_ttl = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
if idempotency_ttl < 86400:
raise ValueError("Idempotency retention must be at least 24 hours")
if not 1 <= ttl <= 2_592_000 or not 1 <= size <= 32:
raise ValueError("Invalid session lifetime or connection pool size")
maximum = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
pixels = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
if not 1 <= maximum <= 10485760 or not 1 <= pixels <= 40000000:
raise ValueError("Invalid upload limits")
return cls(
secret("DATABASE_URL_FILE", "/run/secrets/database_url"),
secret("CACHE_URL_FILE", "/run/secrets/cache_url"),
ttl,
size,
idempotency_ttl,
os.getenv("DATABASE_READ_AUTOCOMMIT", "1") == "1",
os.getenv("S3_ENDPOINT", "http://s3:9000"),
os.getenv("S3_REGION", "us-east-1"),
os.getenv("S3_BUCKET", "messenger-attachments"),
secret("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_access_key_id"),
secret("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_secret_access_key"),
maximum,
pixels,
)
+74
View File
@@ -0,0 +1,74 @@
"""S3 adapter: bounded sockets, least-privilege credentials, no automatic PUT retries."""
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from messenger.domain import DomainError
def unavailable():
return DomainError(503, "unavailable", "Файловое хранилище временно недоступно")
class ObjectStore:
def __init__(self, settings):
self.bucket = settings.s3_bucket
self.client = boto3.client(
"s3",
endpoint_url=settings.s3_endpoint,
region_name=settings.s3_region,
aws_access_key_id=settings.s3_access_key,
aws_secret_access_key=settings.s3_secret_key,
config=Config(
connect_timeout=0.5,
read_timeout=2,
max_pool_connections=8,
retries={"total_max_attempts": 1},
s3={"addressing_style": "path"},
request_checksum_calculation="when_required",
response_checksum_validation="when_required",
),
)
def ready(self):
try:
self.client.head_bucket(Bucket=self.bucket)
except BotoCoreError, ClientError:
raise unavailable() from None
def put(self, key, file, size, media, digest):
try:
self.client.put_object(
Bucket=self.bucket,
Key=key,
Body=file,
ContentLength=size,
ContentType=media,
Metadata={"sha256": digest},
)
except BotoCoreError, ClientError:
raise unavailable() from None
def head(self, key):
try:
return self.client.head_object(Bucket=self.bucket, Key=key)
except ClientError as exc:
if str(exc.response.get("Error", {}).get("Code")) in ("404", "NoSuchKey", "NotFound"):
return None
raise unavailable() from None
except BotoCoreError:
raise unavailable() from None
def download(self, row):
try:
response = self.client.get_object(Bucket=self.bucket, Key=row["object_key"])
if response["ContentLength"] != row["size_bytes"]:
response["Body"].close()
raise unavailable()
return response["Body"]
except BotoCoreError, ClientError:
raise unavailable() from None
def close(self):
self.client.close()
+85
View File
@@ -0,0 +1,85 @@
import base64
from pathlib import Path
from uuid import uuid4
import psycopg
import pytest
from fastapi.testclient import TestClient
from psycopg import sql
from psycopg.conninfo import make_conninfo
from messenger.app import create_app
from messenger.migrate import migrate
from messenger.settings import Settings
ROOT = Path(__file__).resolve().parents[1]
PASSWORD = "Учебный:пароль😀-12345"
@pytest.fixture(scope="session")
def settings():
password = (ROOT / "secrets/postgres_password").read_text().strip()
admin = make_conninfo(
host="127.0.0.1", port=55435, user="messenger", password=password, dbname="messenger"
)
database = "student_tests_" + uuid4().hex
with psycopg.connect(admin, autocommit=True) as conn:
conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(database)))
config = Settings(
make_conninfo(admin, dbname=database),
"redis://127.0.0.1:56382/1",
s3_bucket="messenger-attachments-tests",
s3_access_key=(ROOT / "secrets/s3_test_access_key_id").read_text().strip(),
s3_secret_key=(ROOT / "secrets/s3_test_secret_access_key").read_text().strip(),
)
migrate(config.database_url, ROOT / "migrations")
with pytest.MonkeyPatch.context() as environment:
environment.setenv("S3_ENDPOINT", config.s3_endpoint)
environment.setenv("S3_BUCKET", config.s3_bucket)
environment.setenv("S3_ACCESS_KEY_ID_FILE", str(ROOT / "secrets/s3_test_access_key_id"))
environment.setenv(
"S3_SECRET_ACCESS_KEY_FILE", str(ROOT / "secrets/s3_test_secret_access_key")
)
yield config
with psycopg.connect(admin, autocommit=True) as conn:
conn.execute(sql.SQL("DROP DATABASE {} WITH (FORCE)").format(sql.Identifier(database)))
@pytest.fixture
def client(settings):
with psycopg.connect(settings.database_url) as conn:
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions CASCADE")
with TestClient(create_app(settings), raise_server_exceptions=False) as client:
client.app.state.cache.client.flushdb()
yield client
def basic(username, password=PASSWORD):
return {
"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode()).decode()
}
@pytest.fixture
def actors(client):
result = []
for name in ("alice", "bob", "eve"):
r = client.post(
"/api/v1/users", json={"username": name, "display_name": name, "password": PASSWORD}
)
assert r.status_code == 201
session = client.post("/api/v1/auth/sessions", headers=basic(name))
assert session.status_code == 201
result.append({**r.json(), "token": session.json()["token"]})
return result
@pytest.fixture
def room(client, actors):
response = client.post(
"/api/v1/chats",
headers={"Authorization": "Bearer " + actors[0]["token"]},
json={"title": "Чат 👋", "member_ids": [u["id"] for u in actors[:2]]},
)
assert response.status_code == 201
return response.json()
+299
View File
@@ -0,0 +1,299 @@
import hashlib
import json
import re
import time
from concurrent.futures import ThreadPoolExecutor
from uuid import uuid4
import psycopg
import pytest
from conftest import PASSWORD, ROOT, basic
from fastapi.testclient import TestClient
from messenger.app import create_app
from messenger.auth import basic_credentials
from messenger.migrate import migrate
def headers(actor):
return {"Authorization": "Bearer " + actor["token"], "Idempotency-Key": uuid4().hex}
def error(response, status, code):
assert response.status_code == status, response.text
body = response.json()["error"]
assert set(body) == {"code", "message", "request_id"}
assert body["code"] == code
assert body["request_id"] == response.headers["X-Request-Id"]
assert len(response.headers.get_list("X-Request-Id")) == 1
if status == 401:
assert response.headers["WWW-Authenticate"].startswith(("Basic", "Bearer"))
@pytest.mark.parametrize("body", ["{", "null", "[]", '"text"', '{"username":'])
def test_malformed_json(client, body):
error(
client.post("/api/v1/users", content=body, headers={"Content-Type": "application/json"}),
400,
"invalid_request",
)
@pytest.mark.parametrize(
"change",
[
{"username": "ab"},
{"username": "a" * 33},
{"username": "Upper"},
{"username": "кириллица"},
{"username": "valid\n"},
{"username": 123},
{"username": True},
{"display_name": 123},
{"display_name": None},
{"display_name": " \n\t"},
{"display_name": "😀" * 101},
{"display_name": "\ud800"},
{"password": "short"},
{"password": "😀" * 129},
{"password": 123},
{"password": " " * 12},
{"password": "\ud800" * 12},
{"extra": True},
],
)
def test_strict_registration(client, change):
data = {"username": "valid", "display_name": "Имя", "password": PASSWORD, **change}
response = client.post(
"/api/v1/users", content=json.dumps(data), headers={"Content-Type": "application/json"}
)
error(response, 400, "invalid_request")
def test_unicode_password_hash_and_authentication(client, settings, actors):
for u in actors:
assert "password" not in u and "password_hash" not in u
assert re.fullmatch(r"[A-Za-z0-9_-]{43}", u["token"])
with psycopg.connect(settings.database_url) as conn:
encoded = conn.execute("SELECT password_hash FROM users WHERE username='alice'").fetchone()[
0
]
assert encoded.startswith("$argon2id$") and PASSWORD not in encoded
assert (
conn.execute("SELECT octet_length(token_hash) FROM sessions LIMIT 1").fetchone()[0]
== 32
)
bad = client.post("/api/v1/auth/sessions", headers=basic("alice", "wrong-password"))
absent = client.post("/api/v1/auth/sessions", headers=basic("unknown", "wrong-password"))
error(bad, 401, "unauthorized")
error(absent, 401, "unauthorized")
assert bad.json()["error"]["message"] == absent.json()["error"]["message"]
assert bad.headers["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
for value in (
{},
basic("alice"),
{"X-User-Id": actors[0]["id"]},
{"Authorization": "Bearer unknown"},
):
error(client.get("/api/v1/users/me", headers=value), 401, "unauthorized")
actual = client.get(
"/api/v1/users/me", headers={**headers(actors[0]), "X-User-Id": actors[2]["id"]}
).json()
assert actual["id"] == actors[0]["id"]
error(
client.post("/api/v1/auth/sessions", headers=basic("alice"), json={}),
400,
"invalid_request",
)
@pytest.mark.parametrize("value", [None, "Bearer a", "Basic !", "Basic YWxpY2U=", "Basic /w=="])
def test_invalid_basic(value):
with pytest.raises(Exception) as caught:
basic_credentials(value)
assert caught.value.status == 401
def test_registration_race(client, settings):
def register(_):
return client.post(
"/api/v1/users",
json={"username": "same", "display_name": "Алиса", "password": PASSWORD},
).status_code
with ThreadPoolExecutor(max_workers=2) as pool:
assert sorted(pool.map(register, range(10))) == [201] + [409] * 9
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM users WHERE username='same'").fetchone()[0] == 1
def test_acl_validation_and_transaction_rollback(client, actors, room, settings):
path = "/api/v1/chats/" + room["id"]
for suffix in ("", "/messages"):
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
error(client.get("/api/v1/chats/not-a-uuid"), 401, "unauthorized")
error(
client.get("/api/v1/chats/not-a-uuid", headers=headers(actors[0])), 400, "invalid_request"
)
with psycopg.connect(settings.database_url) as conn:
before = conn.execute("SELECT count(*) FROM chats").fetchone()[0]
for members in ([actors[0]["id"], str(uuid4())], [actors[0]["id"]] * 2, [actors[1]["id"]], []):
error(
client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "rollback", "member_ids": members},
),
400,
"invalid_request",
)
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM chats").fetchone()[0] == before
assert conn.execute("SELECT count(*) FROM chat_members").fetchone()[0] == 2
for body in (
{"text": ""},
{"text": " \t"},
{"text": "😀" * 4001},
{"text": 123},
{"text": None},
{"text": "ok", "sender_id": actors[1]["id"]},
):
error(
client.post(path + "/messages", headers=headers(actors[0]), json=body),
400,
"invalid_request",
)
for query in (
"limit=0",
"limit=101",
"limit=true",
"limit=1.0",
"limit=1&limit=2",
"cursor=",
"cursor=bad!",
"cursor=e30",
"cursor=x&cursor=y",
):
error(
client.get(path + "/messages?" + query, headers=headers(actors[0])),
400,
"invalid_request",
)
def test_concurrent_messages_and_keyset_ties(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
def send(i):
r = client.post(path, headers=headers(actors[i % 2]), json={"text": f"m{i} 😀"})
assert r.status_code == 201
return r.json()
with ThreadPoolExecutor(max_workers=10) as pool:
sent = list(pool.map(send, range(100)))
with psycopg.connect(settings.database_url) as conn:
conn.execute(
"UPDATE messages SET created_at='2026-10-17T12:00:00Z' WHERE chat_id=%s", (room["id"],)
)
for m in sent:
m["created_at"] = "2026-10-17T12:00:00.000000Z"
found, cursor = [], None
while True:
query = {"limit": 17 if cursor else 1}
if cursor:
query["cursor"] = cursor
response = client.get(path, params=query, headers=headers(actors[1]))
assert response.status_code == 200
page = response.json()
found.extend(page["items"])
cursor = page["next_cursor"]
if not cursor:
break
assert len(found) == len({x["id"] for x in found}) == 100
assert found == sorted(sent, key=lambda x: (x["created_at"], x["id"]))
first_cursor = client.get(path + "?limit=1", headers=headers(actors[0])).json()["next_cursor"]
error(
client.get("/api/v1/chats", params={"cursor": first_cursor}, headers=headers(actors[0])),
400,
"invalid_request",
)
def test_cache_hit_flush_and_revocation(client, actors, monkeypatch):
user = actors[0]
url = "/api/v1/users/me"
assert client.get(url, headers=headers(user)).status_code == 200
repository = client.app.state.repository
original = repository.get_user
def must_be_cached(*args):
raise AssertionError("cache hit should avoid fetching immutable user again")
monkeypatch.setattr(repository, "get_user", must_be_cached)
assert client.get(url, headers=headers(user)).status_code == 200
monkeypatch.setattr(repository, "get_user", original)
cache = client.app.state.cache.client
digest = hashlib.sha256(user["token"].encode()).digest()
key = client.app.state.cache.key(digest)
assert 0 < cache.ttl(key) <= 3600
stale = cache.get(key)
cache.flushdb()
assert client.get(url, headers=headers(user)).status_code == 200
assert client.delete("/api/v1/auth/sessions/current", headers=headers(user)).status_code == 204
cache.set(key, stale, ex=3600) # A resurrected stale cache entry must not resurrect a session.
error(client.get(url, headers=headers(user)), 401, "unauthorized")
error(
client.delete("/api/v1/auth/sessions/current", headers=headers(user)), 401, "unauthorized"
)
def test_expiry_and_api_restart(client, actors, settings, monkeypatch, tmp_path):
database_file, cache_file = tmp_path / "database_url", tmp_path / "cache_url"
database_file.write_text(settings.database_url)
cache_file.write_text(settings.cache_url)
monkeypatch.setenv("DATABASE_URL_FILE", str(database_file))
monkeypatch.setenv("CACHE_URL_FILE", str(cache_file))
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
with TestClient(create_app()) as short:
response = short.post("/api/v1/auth/sessions", headers=basic("alice"))
assert response.status_code == 201
token = {"Authorization": "Bearer " + response.json()["token"]}
assert short.get("/api/v1/users/me", headers=token).status_code == 200
with TestClient(create_app(settings)) as restarted:
assert restarted.get("/api/v1/users/me", headers=token).status_code == 200
time.sleep(2.1)
error(restarted.get("/api/v1/users/me", headers=token), 401, "unauthorized")
def test_migrations_idempotent_and_detect_change(client, settings, tmp_path):
migrate(settings.database_url, ROOT / "migrations")
modified = tmp_path / "001_initial.sql"
modified.write_text((ROOT / "migrations/001_initial.sql").read_text() + "\n-- changed")
with pytest.raises(RuntimeError, match="checksum changed"):
migrate(settings.database_url, tmp_path)
def test_logs_and_safe_errors(client, monkeypatch, caplog):
canary = "PRIVATE_BODY_CANARY_123"
with caplog.at_level("INFO", logger="messenger"):
r = client.post(
"/api/v1/users",
json={"username": "logging", "display_name": canary, "password": PASSWORD},
)
assert r.status_code == 201
assert canary not in caplog.text and PASSWORD not in caplog.text
def broken(*args):
raise RuntimeError(canary)
monkeypatch.setattr(client.app.state.authentication, "register", broken)
response = client.post(
"/api/v1/users", json={"username": "broken", "display_name": "x", "password": PASSWORD}
)
error(response, 500, "internal_error")
assert canary not in response.text and canary not in caplog.text
error(
client.post("/api/v1/users", content="{}", headers={"Content-Type": "text/plain"}),
415,
"unsupported_media_type",
)
+244
View File
@@ -0,0 +1,244 @@
import hashlib
import io
import struct
import zlib
from dataclasses import replace
from urllib.parse import quote
from uuid import uuid4
import psycopg
import pytest
from botocore.exceptions import ClientError
from conftest import ROOT
from PIL import Image
from psycopg.rows import dict_row
from test_api import error, headers
from messenger.domain import DomainError
from messenger.reconcile import Reconciler
from messenger.storage import ObjectStore
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
def upload(client, actor, room, content=PNG, media="image/png", name="sample.png"):
return client.post(
f"/api/v1/chats/{room['id']}/attachments?filename={quote(name, safe='')}",
headers={**headers(actor), "Content-Type": media},
content=content,
)
@pytest.fixture
def admin(settings):
store = ObjectStore(
replace(
settings,
s3_access_key=(ROOT / "secrets/s3_root_user").read_text().strip(),
s3_secret_key=(ROOT / "secrets/s3_root_password").read_text().strip(),
)
)
yield store
store.close()
def age(conn, id):
conn.execute(
"UPDATE attachments SET updated_at=clock_timestamp()-interval '10 minutes' WHERE id=%s",
(id,),
)
def test_original_png_jpeg_and_acl(client, actors, room):
jpg = io.BytesIO()
Image.new("RGB", (17, 11), "red").save(jpg, format="JPEG")
for data, media in ((PNG, "image/png"), (jpg.getvalue(), "image/jpeg")):
response = upload(client, actors[0], room, data, media, "Фото 1.png")
assert response.status_code == 201, response.text
value = response.json()
assert value["status"] == "ready" and value["variants"] == []
assert value["error_code"] is None and value["sha256"] == hashlib.sha256(data).hexdigest()
assert value["size_bytes"] == len(data) and value["content_type"] == media
assert "object_key" not in value
path = "/api/v1/attachments/" + value["id"]
assert client.get(path, headers=headers(actors[1])).json() == value
download = client.get(path + "/content", headers=headers(actors[1]))
assert download.status_code == 200 and download.content == data
assert download.headers["Content-Type"] == media
for suffix in ("", "/content"):
error(client.get(path + suffix, headers=headers(actors[2])), 404, "not_found")
error(client.get(path + suffix), 401, "unauthorized")
error(
client.get(path + "/content?variant=thumbnail", headers=headers(actors[0])),
400,
"invalid_request",
)
@pytest.mark.parametrize("name", ["", "../a.png", "a\\b.png", "a\x00.png", "\n.png", "x" * 256])
def test_unsafe_names(client, actors, room, name):
error(upload(client, actors[0], room, name=name), 400, "invalid_request")
@pytest.mark.parametrize(
"data,media",
[
(b"", "image/png"),
(b"not a file", "image/png"),
(PNG[:30], "image/png"),
(PNG, "image/jpeg"),
(PNG, "text/plain"),
],
)
def test_invalid_images(client, actors, room, data, media):
error(upload(client, actors[0], room, data, media), 415, "unsupported_media_type")
def test_byte_limit_without_content_length_and_pixel_bomb(client, actors, room):
error(upload(client, actors[0], room, b"x" * (10485760 + 1)), 413, "payload_too_large")
stream = (b"x" * 1048576 for _ in range(11))
error(upload(client, actors[0], room, stream), 413, "payload_too_large")
bomb = bytearray(PNG)
bomb[16:24] = struct.pack(">II", 10000, 4001)
bomb[29:33] = struct.pack(">I", zlib.crc32(bomb[12:29]) & 0xFFFFFFFF)
error(upload(client, actors[0], room, bytes(bomb)), 413, "payload_too_large")
def test_attachment_message_relations_and_replay(client, actors, room):
att = upload(client, actors[0], room).json()
path = f"/api/v1/chats/{room['id']}/messages"
h = headers(actors[1])
body = {"text": "", "attachment_ids": [att["id"]]}
result = client.post(path, headers=h, json=body)
assert result.status_code == 201 and result.json()["attachment_ids"] == [att["id"]]
assert client.post(path, headers=h, json=body).json() == result.json()
assert client.get(path, headers=headers(actors[0])).json()["items"] == [result.json()]
error(
client.post(
path,
headers=headers(actors[0]),
json={"text": "", "attachment_ids": [att["id"], att["id"]]},
),
400,
"invalid_request",
)
error(
client.post(
path, headers=headers(actors[0]), json={"text": "", "attachment_ids": [str(uuid4())]}
),
404,
"not_found",
)
other = client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "other", "member_ids": [actors[0]["id"]]},
).json()
error(
client.post(f"/api/v1/chats/{other['id']}/messages", headers=headers(actors[0]), json=body),
404,
"not_found",
)
legacy = headers(actors[0])
one = client.post(path, headers=legacy, json={"text": "legacy"})
two = client.post(path, headers=legacy, json={"attachment_ids": [], "text": "legacy"})
assert one.status_code == 201 and two.json() == one.json()
def test_scoped_credentials_cannot_delete_or_access_other_bucket(client, settings):
store = ObjectStore(settings)
try:
with pytest.raises(ClientError) as denied:
store.client.delete_object(Bucket=store.bucket, Key="originals/" + str(uuid4()))
assert denied.value.response["ResponseMetadata"]["HTTPStatusCode"] == 403
with pytest.raises(ClientError) as denied:
store.client.list_objects_v2(Bucket="unrelated-bucket")
assert denied.value.response["ResponseMetadata"]["HTTPStatusCode"] == 403
finally:
store.close()
def test_db_failure_after_put_is_recoverable_with_dry_run(
client, actors, room, settings, monkeypatch, admin
):
repo = client.app.state.repository
def fail(*args):
raise RuntimeError("injected after durable PUT")
monkeypatch.setattr(repo, "finish_attachment", fail)
error(upload(client, actors[0], room), 500, "internal_error")
store = client.app.state.attachments.store
with psycopg.connect(settings.database_url, autocommit=True, row_factory=dict_row) as conn:
row = conn.execute("SELECT * FROM attachments").fetchone()
assert row["storage_state"] == "pending" and store.head(row["object_key"]) is not None
assert Reconciler(conn, admin, apply=True).run()["counts"].get("restore_ready", 0) == 0
age(conn, row["id"])
dry = Reconciler(conn, admin).run()
assert dry["counts"]["restore_ready"] == 1
assert (
conn.execute("SELECT storage_state FROM attachments").fetchone()["storage_state"]
== "pending"
)
assert Reconciler(conn, admin, apply=True).run()["counts"]["restore_ready"] == 1
assert Reconciler(conn, admin, apply=True).run()["actions"] == []
row = conn.execute("SELECT * FROM attachments").fetchone()
assert row["storage_state"] == "ready"
assert (
client.get(
"/api/v1/attachments/" + str(row["id"]) + "/content", headers=headers(actors[1])
).content
== PNG
)
def test_failed_put_and_missing_used_object(client, actors, room, settings, admin, monkeypatch):
store = client.app.state.attachments.store
original = store.put
def fail(*args):
raise DomainError(503, "unavailable", "injected transport failure")
monkeypatch.setattr(store, "put", fail)
error(upload(client, actors[0], room), 503, "unavailable")
with psycopg.connect(settings.database_url, autocommit=True, row_factory=dict_row) as conn:
pending = conn.execute("SELECT id FROM attachments").fetchone()["id"]
age(conn, pending)
assert Reconciler(conn, admin, apply=True).run()["counts"]["delete_abandoned_intent"] == 1
monkeypatch.setattr(store, "put", original)
att = upload(client, actors[0], room).json()
response = client.post(
f"/api/v1/chats/{room['id']}/messages",
headers=headers(actors[0]),
json={"text": "used", "attachment_ids": [att["id"]]},
)
assert response.status_code == 201
row = conn.execute("SELECT * FROM attachments WHERE id=%s", (att["id"],)).fetchone()
age(conn, row["id"])
admin.client.delete_object(Bucket=store.bucket, Key=row["object_key"])
assert Reconciler(conn, admin).run()["counts"]["mark_missing"] == 1
assert (
conn.execute("SELECT storage_state FROM attachments").fetchone()["storage_state"]
== "ready"
)
assert Reconciler(conn, admin, apply=True).run()["counts"]["mark_missing"] == 1
error(
client.get("/api/v1/attachments/" + att["id"], headers=headers(actors[0])),
409,
"conflict",
)
assert conn.execute("SELECT count(*) AS n FROM message_attachments").fetchone()["n"] == 1
admin.put(
row["object_key"],
io.BytesIO(PNG),
len(PNG),
"image/png",
hashlib.sha256(PNG).hexdigest(),
)
assert Reconciler(conn, admin, apply=True).run()["counts"]["restore_ready"] == 1
assert (
client.get(
"/api/v1/attachments/" + att["id"] + "/content", headers=headers(actors[0])
).content
== PNG
)
+137
View File
@@ -0,0 +1,137 @@
import json
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from uuid import uuid4
import psycopg
import pytest
from fastapi.testclient import TestClient
from test_api import error, headers
from messenger.app import create_app
@pytest.mark.parametrize("key", [None, "", "a b", "/", "a" * 129])
def test_message_key_required(client, actors, room, key):
h = {"Authorization": "Bearer " + actors[0]["token"]}
if key is not None:
h["Idempotency-Key"] = key
error(
client.post(
f"/api/v1/chats/{room['id']}/messages", headers=h, json={"text": "key validation"}
),
400,
"invalid_request",
)
def test_replay_scope_conflict_and_fresh_authorization(client, actors, room):
path = f"/api/v1/chats/{room['id']}/messages"
key = "Scope.Key:123"
h = {**headers(actors[0]), "Idempotency-Key": key}
first = client.post(path, headers=h, json={"text": "exact whitespace \n"})
assert first.status_code == 201
repeat = client.post(
path,
headers={**h, "Content-Type": "application/json"},
content=json.dumps({"text": "exact whitespace \n"}, indent=3),
)
assert repeat.status_code == 201 and repeat.json() == first.json()
error(client.post(path, headers=h, json={"text": "different"}), 409, "conflict")
outsider = {**headers(actors[2]), "Idempotency-Key": key}
error(
client.post(path, headers=outsider, json={"text": "exact whitespace \n"}), 404, "not_found"
)
other_user = {**headers(actors[1]), "Idempotency-Key": key}
own = client.post(path, headers=other_user, json={"text": "another author"})
assert own.status_code == 201 and own.json()["id"] != first.json()["id"]
chat = client.post(
"/api/v1/chats",
headers=headers(actors[0]),
json={"title": "another", "member_ids": [actors[0]["id"]]},
).json()
other_chat = client.post(
f"/api/v1/chats/{chat['id']}/messages", headers=h, json={"text": "different chat"}
)
assert other_chat.status_code == 201 and other_chat.json()["id"] != first.json()["id"]
assert client.delete("/api/v1/auth/sessions/current", headers=h).status_code == 204
error(client.post(path, headers=h, json={"text": "exact whitespace \n"}), 401, "unauthorized")
def test_concurrent_replay_across_independent_apps(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
with TestClient(create_app(settings)) as second:
assert second.app.state.instance_id != client.app.state.instance_id
def send(i):
c = client if i % 2 else second
r = c.post(path, headers=h, json={"text": "same concurrent body"})
assert r.status_code == 201
return r.json(), r.headers["X-Instance-Id"]
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(send, range(30)))
assert len({item[0]["id"] for item in results}) == 1
assert len({item[1] for item in results}) == 2
assert len(client.get(path, headers=headers(actors[1])).json()["items"]) == 1
def test_side_effect_and_result_rollback_together(client, actors, room, settings, monkeypatch):
repository = client.app.state.repository
original = repository._save_request
def fail_after_message_insert(*args):
raise RuntimeError("injected between message and result")
monkeypatch.setattr(repository, "_save_request", fail_after_message_insert)
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
error(
client.post(path, headers=h, json={"text": "retry after rollback"}), 500, "internal_error"
)
with psycopg.connect(settings.database_url) as conn:
assert conn.execute("SELECT count(*) FROM messages").fetchone()[0] == 0
assert conn.execute("SELECT count(*) FROM message_requests").fetchone()[0] == 0
monkeypatch.setattr(repository, "_save_request", original)
result = client.post(path, headers=h, json={"text": "retry after rollback"})
assert result.status_code == 201
assert (
client.post(path, headers=h, json={"text": "retry after rollback"}).json() == result.json()
)
def test_expired_key_can_be_reused_without_deleting_old_message(client, actors, room, settings):
path = f"/api/v1/chats/{room['id']}/messages"
h = {**headers(actors[0]), "Idempotency-Key": uuid4().hex}
first = client.post(path, headers=h, json={"text": "old"}).json()
with psycopg.connect(settings.database_url) as conn:
conn.execute(
"UPDATE message_requests SET created_at=clock_timestamp()-interval '2 days', "
"expires_at=clock_timestamp()-interval '1 day'"
)
second = client.post(path, headers=h, json={"text": "new"})
assert second.status_code == 201 and second.json()["id"] != first["id"]
assert len(client.get(path, headers=headers(actors[0])).json()["items"]) == 2
@pytest.mark.parametrize("optimized", [True, False])
def test_read_modes_preserve_public_result(client, actors, room, settings, optimized):
with TestClient(create_app(replace(settings, read_autocommit=optimized))) as c:
c.get("/api/v1/users/me", headers=headers(actors[0]))
pool = c.app.state.repository.pool
before = pool.get_stats().get("requests_num", 0)
response = c.get(f"/api/v1/chats/{room['id']}/messages", headers=headers(actors[0]))
assert response.status_code == 200 and response.json()["items"] == []
assert pool.get_stats()["requests_num"] - before == 3
def test_instance_header_covers_health_errors_and_success(client):
values = [
client.get("/health/live"),
client.get("/health/ready"),
client.get("/api/v1/users/me"),
client.get("/not-a-route"),
]
assert len({r.headers["X-Instance-Id"] for r in values}) == 1
assert all(len(r.headers.get_list("X-Instance-Id")) == 1 for r in values)
Generated
+1075
View File
File diff suppressed because it is too large Load Diff